diff --git a/.circleci/config.yml b/.circleci/config.yml
index 5f37b3b1e9..8964490e3b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -2,7 +2,8 @@ version: 2.1
 jobs:
   datapipeline-build:
     working_directory: ~/dp/data-pipeline-flink
-    machine: true
+    machine:
+      image: ubuntu-2004:202008-01
     steps:
       - checkout:
           path: ~/dp
@@ -22,4 +23,4 @@ workflows:
   build_and_test:
     jobs:
       - datapipeline-build
-#testing quality gate
\ No newline at end of file
+#testing  quality gate
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
new file mode 100644
index 0000000000..2bd4425875
--- /dev/null
+++ b/.github/pull_request_template.md
@@ -0,0 +1,32 @@
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+### Type of change
+
+Please choose appropriate options.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+### How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes in the below checkboxes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Ran Test A
+- [ ] Ran Test B
+
+**Test Configuration**:
+* Software versions:
+* Hardware versions:
+
+### Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules
\ No newline at end of file
diff --git a/.github/workflows/jira-description-action.yml b/.github/workflows/jira-description-action.yml
new file mode 100644
index 0000000000..a51bcc59fa
--- /dev/null
+++ b/.github/workflows/jira-description-action.yml
@@ -0,0 +1,16 @@
+name: jira-description-action
+on:
+  pull_request_target:
+    types: [opened, labeled]
+jobs:
+  add-jira-description:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: project-sunbird/jira-description-action@v0.4.0
+        name: jira-description-action
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          jira-token: ${{ secrets.JIRA_TOKEN }}
+          jira-base-url: ${{ secrets.JIRA_BASE_URL }}
+          fail-when-jira-issue-not-found: ${{ secrets.FAIL_WHEN_JIRA_ISSUE_NOT_FOUND }}
+          use: both
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index d222fe92b9..9fcc736131 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ database.yml
 .idea
 tmp/
 kibana-auth/node_modules/
+kubernetes/helm_charts/*/values.yaml
 
 # DevOps
 *.backup
diff --git a/ansible/artifacts-download.yml b/ansible/artifacts-download.yml
index e75c4a941a..fcf58787f5 100644
--- a/ansible/artifacts-download.yml
+++ b/ansible/artifacts-download.yml
@@ -3,8 +3,40 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-download-azure
+  tasks:
+    - name: download artifact from azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-download.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: download artifact from gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: download.yml
+      vars:
+        gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+        gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: download artifact from aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: download.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"
diff --git a/ansible/artifacts-upload.yml b/ansible/artifacts-upload.yml
index 41ef0387ef..305492afc2 100644
--- a/ansible/artifacts-upload.yml
+++ b/ansible/artifacts-upload.yml
@@ -3,8 +3,41 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-upload-azure
+  tasks:
+    - name: upload artifact to azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-upload.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        container_public_access: "off"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: upload artifact to gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+        gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: upload artifact to aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"
diff --git a/ansible/azure-hdinsight-spark.provision.yml b/ansible/azure-hdinsight-spark.provision.yml
new file mode 100644
index 0000000000..3cc166daf3
--- /dev/null
+++ b/ansible/azure-hdinsight-spark.provision.yml
@@ -0,0 +1,18 @@
+- hosts: local
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - azure-hdinsight-spark-cluster
+  tags:
+    - copy-script
+    
+- hosts: spark-hdinsight-cluster
+  become: yes
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - provision-azure-spark-cluster
+  tags:
+    - spark-provision
diff --git a/ansible/cassandra-backup.yml b/ansible/cassandra-backup.yml
index 204653b815..c9e4d37d22 100644
--- a/ansible/cassandra-backup.yml
+++ b/ansible/cassandra-backup.yml
@@ -2,8 +2,5 @@
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
   roles:
    - cassandra-backup
diff --git a/ansible/cassandra-restore.yml b/ansible/cassandra-restore.yml
index 9fdf46d642..f8538b83c0 100644
--- a/ansible/cassandra-restore.yml
+++ b/ansible/cassandra-restore.yml
@@ -2,9 +2,5 @@
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-
   roles:
    - cassandra-restore
diff --git a/ansible/dp_kafka_setup.yml b/ansible/dp_kafka_setup.yml
index c48a5c4645..a5ea2ccc0f 100644
--- a/ansible/dp_kafka_setup.yml
+++ b/ansible/dp_kafka_setup.yml
@@ -4,7 +4,7 @@
   pre_tasks:
     - name: Registering kafka_id
       set_fact:
-        kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+        kafka_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
     - name: Print kafka_id
       debug:
         var: kafka_id
diff --git a/ansible/dp_yarn_provision.yml b/ansible/dp_yarn_provision.yml
deleted file mode 100644
index 28225e9c8a..0000000000
--- a/ansible/dp_yarn_provision.yml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-- hosts: yarn
-  gather_facts: false
-  become: true
-  tasks:
-    - name: Installaing python
-      raw: bash -c "test -e /usr/bin/python" || (apt update && apt install python -y)
-
-- hosts: yarn
-  become: yes
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
-  tasks:
-    - name: Create group
-      group: 
-        name: hadoop 
-        state: present
-    - name: Create user
-      user: 
-        name: hduser 
-        comment: "hduser" 
-        group: hadoop 
-        groups: sudo 
-        shell: /bin/bash
-
-- name: Install samza job server
-  hosts: "yarn-master"
-  become: yes
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
-  roles:
-    - jdk-1.8.0_121
-    - yarn
-    - samza-job-server
-
-- name: Install java on all yarn slaves
-  hosts: "yarn-slave"
-  become: yes
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
-  remote_user: hduser
-  roles:
-    - jdk-1.8.0_121
diff --git a/ansible/druid-ingestion.yml b/ansible/druid-ingestion.yml
index bff64e86a4..4774cda065 100644
--- a/ansible/druid-ingestion.yml
+++ b/ansible/druid-ingestion.yml
@@ -1,6 +1,36 @@
 ---
+- hosts: localhost
+  tasks:
+   - name: set_fact
+     set_fact:
+       single_cluster: true
+     tags: populate_var
 
-- hosts: overlord
+   - name: check for hosts
+     set_fact:
+       single_cluster: false
+     when: "'rollup-overlord' in groups"
+     tags: populate_var
+
+- hosts: rollup-overlord
+  become: yes
+  roles:
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ rollup_ingestion_task_names }}"
+        overlord_host: "{{ rollup_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+
+- hosts: raw-overlord
   become: yes
   roles:
-     - druid-ingestion
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }},{{ rollup_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: hostvars['localhost']['single_cluster']
diff --git a/ansible/druid-provision.yml b/ansible/druid-provision.yml
index 031f1d8fe5..b0425ecf98 100644
--- a/ansible/druid-provision.yml
+++ b/ansible/druid-provision.yml
@@ -1,38 +1,18 @@
 -
   name: Install Java
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
   roles: 
-    - jdk-1.8.0_121
+    - {role: jdk-1.8.0_121, when: "'java' in deploy"}
 
 -
   name: Install java and zookeeper cluster for druid
-  hosts: raw-zookeeper
+  hosts: "{{ remote }}-zookeeper"
   become: yes
   roles:
-    - {role: zookeeper-upgrade}
+    - {role: zookeeper-upgrade, when: "'zookeeper' in deploy"}
   vars:
-    zookeeper_group: "{{ groups['raw-zookeeper'] }}"
-
-#-
-#  name: Install postgres on druid cluster
-#  hosts: druid-postgres
-#  vars_files:
-#    - "{{inventory_dir}}/secrets.yml"
-#  become: yes
-#  roles:
-#    - {role: postgres-provision }
-#  vars:
-#    postgres_address_space: "0.0.0.0/0"
-#    postgresql_databases:
-#      - name: druid
-#      - name: graphite
-#    postgresql_users:
-#      - name: druid
-#        password: "{{ dp_vault_druid_postgress_pass }}"
-#      - name: graphite 
-#        password: "{{ dp_vault_druid_postgress_graphite_pass }}"
-
+    zookeeper_group: "{{ groups[remote+'-zookeeper'] }}"
 
 # Create postgres databases and users
 
@@ -43,14 +23,10 @@
   vars:
     postgresql_databases:
       - name: druid
-      - name: graphite
     postgresql_users:
       - name: druid
         db: druid
         password: "{{ dp_vault_druid_postgress_pass }}"
-      - name: graphite
-        db: graphite
-        password: "{{ dp_vault_druid_postgress_graphite_pass }}"     
   tasks:
     - name: Ensure database is created
       postgresql_db: name="{{ item.name }}" \
@@ -75,24 +51,15 @@
                   db="{{ item.db }}"
       with_items: "{{ postgresql_users }}"
 
--
-  name : Install graphite on druid
-  hosts: raw-graphite
-  become: yes
-  roles:
-    - {role: druid_graphite}
-    
 -
   name: Install role druid
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
   roles:
-    - {role: analytics-druid, when: "'raw-coordinator' in group_names", druid_role: 'coordinator', service: 'coordinator' }
-    - {role: analytics-druid, when: "'raw-overlord' in group_names", druid_role: 'overlord', service: 'overlord'}
-    - {role: analytics-druid, when: "'raw-broker' in group_names", druid_role: 'broker', service: 'broker'}
-    - {role: analytics-druid, when: "'raw-historical' in group_names", druid_role: 'historical', service: 'historical'}
-    - {role: analytics-druid, when: "'raw-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' }
-  vars:
-    druid_postgres_db: "druid"
-    druid_zookeeper_host: "{{ groups['raw-zookeeper']|join(':2181,')}}:2181"
-    druid_graphite_host: "{{ groups['raw-graphite'][0] }}"
+    - {role: analytics-druid, when: "'{{ remote }}-coordinator' in group_names and 'coordinator' in deploy", druid_role: 'coordinator', service: 'coordinator' }
+    - {role: analytics-druid, when: "'{{ remote }}-overlord' in group_names and 'overlord' in deploy", druid_role: 'overlord', service: 'overlord'}
+    - {role: analytics-druid, when: "'{{ remote }}-broker' in group_names and 'broker' in deploy", druid_role: 'broker', service: 'broker'}
+    - {role: analytics-druid, when: "'{{ remote }}-historical' in group_names and 'historical' in deploy", druid_role: 'historical', service: 'historical'}
+    - {role: analytics-druid, when: "'{{ remote }}-middlemanager' in group_names and 'mm' in deploy", druid_role: 'middlemanager', service: 'middleManager' }
+    - {role: analytics-druid, when: "'{{ remote }}-router' in group_names and 'router' in deploy", druid_role: 'router', service: 'router' }
+
diff --git a/ansible/druid-rollup-provision.yml b/ansible/druid-rollup-provision.yml
index 18ef3e4b88..805d5bb75d 100644
--- a/ansible/druid-rollup-provision.yml
+++ b/ansible/druid-rollup-provision.yml
@@ -91,5 +91,6 @@
     - {role: analytics-druid, when: "'druid-rollup-broker' in group_names", druid_role: 'broker', service: 'broker'}
     - {role: analytics-druid, when: "'druid-rollup-historical' in group_names", druid_role: 'historical', service: 'historical'}
     - {role: analytics-druid, when: "'druid-rollup-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' }
+    - {role: analytics-druid, when: "'druid-rollup-router' in group_names", druid_role: 'router', service: 'router' }
   vars:
     druid_postgres_db: "druid-rollup"
diff --git a/ansible/druid-run-report.yml b/ansible/druid-run-report.yml
index 9e1bf9a51d..a774d51a52 100644
--- a/ansible/druid-run-report.yml
+++ b/ansible/druid-run-report.yml
@@ -1,6 +1,8 @@
 -
   name: Run Druid Job Report
   hosts: spark
+  vars_files:
+       - "{{inventory_dir}}/secrets.yml"
   become: yes
   roles: 
     - analytics-run-report
diff --git a/ansible/druid-summary-monthly-ingestion.yml b/ansible/druid-summary-monthly-ingestion.yml
new file mode 100644
index 0000000000..0674884cc0
--- /dev/null
+++ b/ansible/druid-summary-monthly-ingestion.yml
@@ -0,0 +1,5 @@
+---
+- hosts: rollup-overlord
+  become: yes
+  roles:
+     - druid-summary-monthly-ingestion
diff --git a/ansible/druid_anomaly_detection.yml b/ansible/druid_anomaly_detection.yml
new file mode 100644
index 0000000000..db10993b06
--- /dev/null
+++ b/ansible/druid_anomaly_detection.yml
@@ -0,0 +1,4 @@
+- hosts: spark
+  become: yes
+  roles:
+    - druid-anomaly-detection
\ No newline at end of file
diff --git a/ansible/es_backup.yml b/ansible/es_backup.yml
index 874fbd40fb..3538de2c08 100644
--- a/ansible/es_backup.yml
+++ b/ansible/es_backup.yml
@@ -1,9 +1,15 @@
 - hosts: telemetry-search-cluster
   become: yes
   vars_files:
-    - ['{{inventory_dir}}/secrets.yml']
+    - ['{{inventory_dir}}/secrets.yml','secrets/{{env}}.yml']
   roles:
-    - es-azure-snapshot
+    - role: es-azure-snapshot
+      when: cloud_service_provider == "azure"
+    - role: es-s3-snapshot
+      when: cloud_service_provider == "aws"
+    - role: es-gcs-snapshot
+      when: cloud_service_provider == "gcloud"
+    - role: es5-snapshot-purge
   tags:
     - es_backup
-  run_once: true
\ No newline at end of file
+  run_once: true
diff --git a/ansible/es_telemetry_cluster_setup.yml b/ansible/es_telemetry_cluster_setup.yml
deleted file mode 100644
index bdc2998826..0000000000
--- a/ansible/es_telemetry_cluster_setup.yml
+++ /dev/null
@@ -1,36 +0,0 @@
----
-- hosts: telemetry-search-cluster
-  name: Setup telemetry ES cluster
-  remote_user: ecosystem
-  become: yes
-  pre_tasks:
-    - name: Uncompressing and copying to system path
-      unarchive:
-        src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz
-        dest: /opt/
-        remote_src: yes
-        keep_newer: yes
-      become: yes
-      register: tar
-    - name: Updating java jre to system path
-      become: yes
-      shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999
-    - name: Registering node name
-      set_fact:
-        es_instance_name: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}es-node-{{ loop.index }}{% endif %}{% endfor %}"
-
-  roles:
-    - { role: es6, 
-        es_config: {
-          cluster.name: "elasticsearch-telemetry-search",
-          discovery.zen.ping.unicast.hosts: "{{ groups['telemetry-search-cluster'] }}",
-          http.port: 9200,
-          transport.tcp.port: 9300,
-          node.data: "{{ es_etc_node_data | default('true') }}",
-          node.master: "{{ es_etc_node_master | default('true') }}",
-          bootstrap.memory_lock: true,
-        },
-        es_heap_size: "2g",
-        es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['telemetry-search-cluster'] }}",
-        es_etc_cluster_name: "elasticsearch_telemetry_search"
-      }
diff --git a/ansible/graphite-exporter.yml b/ansible/graphite-exporter.yml
new file mode 100644
index 0000000000..7c4d99edad
--- /dev/null
+++ b/ansible/graphite-exporter.yml
@@ -0,0 +1,6 @@
+- hosts: druid-raw
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - graphite-exporter
\ No newline at end of file
diff --git a/ansible/influxdb_backup.yml b/ansible/influxdb_backup.yml
index 9f4b40b2a9..77e8604a5c 100644
--- a/ansible/influxdb_backup.yml
+++ b/ansible/influxdb_backup.yml
@@ -4,9 +4,5 @@
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
   become: yes
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
   roles:
-    - azure-cli
     - influxdb_backup
diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml
index dffa6d3877..13114d01d9 100644
--- a/ansible/inventory/env/group_vars/all.yml
+++ b/ansible/inventory/env/group_vars/all.yml
@@ -1,4 +1,3 @@
-###################### DP ############################
 # Not used
 metrics_search_params_bucket: "dev-data-store"
 log4j_appender_kafka_topic: "{{env}}.telemetry.backend"
@@ -8,46 +7,27 @@ log4j_appender_kafka_topic: "{{env}}.telemetry.backend"
 # What's the implication
 # Is it azure blob or s3 bucket??
 channel_data_exhaust_bucket: dev-data-store
-
-
-#azure_account_name: "{{dp_azure_account_name}}"
 secrets_path: '{{inventory_dir}}/secrets.yml'
 artifacts_container: "{{dp_vault_artifacts_container}}"
-#azure_storage_secret: "{{sunbird_private_storage_account_key}}"
 
 report_azure_account_name: "{{sunbird_private_storage_account_name}}"
 report_azure_storage_secret: "{{sunbird_private_storage_account_key}}"
-#azure_storage_secret: "{{dp_vault_azure_account_key}}"
-#monasca_log_level: ERROR
-resourcemanager: "{{ groups['yarn-master'][0] }}"
-yarn_slaves: "{{ groups['yarn-slave']}}"
-monitor_yarn_url: "http://{{ resourcemanager }}:8088/ws/v1/cluster/apps?states=RUNNING"
-
-# Artifact upload and download vars
-#artifact_azure_account_name: "{{dp_azure_account_name}}"
-#artifact_azure_account_key: "{{dp_vault_azure_account_key}}"
-
-# Artifact upload and download vars
-#artifact_azure_account_name: "{{dp_azure_account_name}}"
-#artifact_azure_account_key: "{{dp_vault_azure_account_key}}"
 
 redis_host: "{{ groups['redis'][0] }}"
 metadata_redis_host: "{{ groups['redis'][0] }}"
-searchServiceAuthorizationToken: "{{ dp_search_service_authorization_token }}"
-
+metadata2_redis_host: "{{ groups['redis'][0] }}"
+searchServiceAuthorizationToken: "{{ sunbird_api_auth_token }}"
 
 # Learning Service organization and location search endpoints
 channelSearchServiceEndpoint: "{{ proto}}://{{domain_name}}/api/org/v1/search"
 locationSearchServiceEndpoint: "{{ proto}}://{{domain_name}}/api/data/v1/location/search"
 
-__yarn_host__: "{{ groups['yarn-master'][0] }}"
-search_service_endpoint: "{{proto}}://{{domain_name}}/api/search/v2/search"
-yarn_es_hosts: "{{ groups['telemetry-search-cluster'] | join(',') }}"
 env_name: "{{env}}"
 kafka_brokers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
 zookeepers: "{{groups['zookeeper']|join(':2181,')}}:2181"
 # Ingestion cluster
 ingestion_kafka_brokers: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+secor_ingestion_kafka_brokers: "{{groups['processing-cluster-kafka']|join(',')}}"
 ingestion_zookeepers: "{{groups['zookeeper']|join(':2181,')}}:2181"
 
 telemetry_schema_directory: /etc/{{env}}/telemetry
@@ -56,26 +36,19 @@ schema_repo_url: https://github.com/project-sunbird/sunbird-data-pipeline.git
 # Create learningall group with LP ips
 cassandra_host:  "{{ groups['cassandra'][0] }}"
 core_cassandra_host: "{{ groups['core-cassandra'][0] }}"
-content_to_vec_url: http://{{ groups['analytics-api'][0] }}:9000/content-to-vec
-
-kibana_base_path : "/pipeline-dashboard"
-
-################ LPA ###########################
+lp_cassandra_host: "{{ groups['lp-cassandra'][0] }}"
+report_cassandra_host: "{{ groups['report-cassandra'][0] }}"
 
 analytics_user: analytics
 analytics_group: analytics
-#azure_storage_account: "{{dp_azure_account_name}}"
-#azure_storage_key: "{{dp_vault_azure_account_key}}"
-
-
 analytics_user_home: /home/{{ analytics_user }}
 sbin_path: "{{ analytics_user_home }}/sbin"
 
-
 # Secor vars
 secor:
   properties: ['secor.azure', 'secor.common', 'secor', 'secor.partition', 'log4j']
   artifact_dir: /mount/secor
+  artifact_ver: "0.29"
   azure:
     account_name: "{{sunbird_private_storage_account_name}}"
     account_key: "{{sunbird_private_storage_account_key}}"
@@ -89,7 +62,7 @@ secor:
 postgresql_databases:
   - name: analytics
     owner: analytics
-    
+
 postgresql_users:
   - name: analytics
     password: "{{dp_vault_pgdb_password}}"
@@ -108,7 +81,7 @@ postgres_address_space: 0.0.0.0/0 # Postgres trust address space
 
 lp_composite_search: "http://{{ groups['composite-search-cluster'][0] }}:9200" # Composite Cluster ip of LP
 lp_composite_search_host: "{{ groups['composite-search-cluster'][0] }}"
-lp_search: "http://{{ groups['search'][0] }}:9000"
+lp_search: "http://{{private_ingressgateway_ip}}/search"
 lp_url: http://{{ groups['learning'][0] }}:8080/learning-service
 service:
   search:
@@ -119,29 +92,19 @@ cassandra_hierarchy_store_prefix: "{{env}}_"
 data_exhaust_token: "{{dp_vault_data_exhaust_token}}"
 default_channel: "{{default_org_hash_id}}"
 
-#### backup storage secret
-#backup_azure_storage_account_name: "{{dp_azure_account_name}}"
-#backup_azure_storage_access_key: "{{dp_vault_azure_account_key}}"
-
-## es backup
-es_snapshot_host: "{{ groups['telemetry-search-cluster'][0] }}"
-snapshot_base_path: telemetrysearch
-
-
 # sunbird learner service
 sunbird_es_host: "{{groups['core-es']| join(',')}}"
+single_node_es_host: "{{ groups['core-es'][0] }}"
 
 ### Druid related vars
-overlord_host: "{{ groups['overlord'][0] }}"
+raw_overlord_host: "{{ groups['raw-overlord'][0] }}"
+rollup_overlord_host: "{{ groups['rollup-overlord'][0] }}"
 
 ### Datapipeline Custom monitoring related vars ###
 influxdb: "{{ groups['influxdb'][0] }}"
 CONTAINER_NAME_SAMZA: samza-logs
 script_path: /usr/local/hadoop
 
-#### samza jobs deployment related vars ####
-samza_tar_files_localpath: roles/samza-jobs/defaults
-
 job_names:
   DeDuplication_1:
       job_file_name: 'de-duplication'
@@ -172,7 +135,7 @@ job_names:
   ContentCacheUpdater_1:
       job_file_name: 'content-cache-updater'
   ShareEventsFlattener_1:
-      job_file_name: 'share-events-flattener'    
+      job_file_name: 'share-events-flattener'
 
 druid_ingestion_specs:
   telemetry-events:
@@ -186,7 +149,37 @@ druid_ingestion_specs:
 
 #Druid Proxy APi service
 sunbird_druid_broker_host: "http://{{ groups['raw-broker'][0] }}"
-sunbird_learner_service_url: "{{swarm_manager_lb_ip}}:9000"
+sunbird_learner_service_url: "http://{{private_ingressgateway_ip}}/learner"
 
 location_search_url: "{{ domain_name }}/api/data/"
-location_search_token: "Bearer {{ dp_search_service_authorization_token }}"
+location_search_token: "Bearer {{ sunbird_api_auth_token }}"
+
+druid_report_url_endpoint : "{{ proto}}://{{domain_name}}/api/data/v1/report/jobs"
+druid_report_url : "{{ proto}}://{{domain_name}}/api/data/v1/"
+druid_report_token : "Bearer {{ sunbird_api_auth_token }}"
+
+#redis multiprocess config
+content_port: 6379
+device_port: 6380
+user_port: 6381
+dialcode_port: 6382
+
+### kafka and zookeeper ip vars and being used in kafka topiccreation role.
+ingestion_zookeeper_ip: "{{ groups['ingestion-cluster-zookeeper'][0] }}"
+processing_zookeeper_ip: "{{ groups['processing-cluster-zookeepers'][0] }}"
+### docker image pull secrets name and this will be created in kubernetes
+imagepullsecrets: "{{env}}-registry-secret"
+dialcode_host: "http://{{private_ingressgateway_ip}}/dial"
+dialcode_endpoint: "dialcode/v3/read/"
+
+exhaust_job_assessment_primary_category: "Practice Question Set"
+etb_dialcode_list_druid_length: 70
+# s3 storage config
+s3_storage_key: "{{ sunbird_private_s3_storage_key }}"
+s3_storage_secret: "{{ sunbird_private_s3_storage_secret }}"
+s3_storage_endpoint: ""
+s3_request_signature_version: AWS4-HMAC-SHA256
+s3_path_style_access: false
+s3_https_only: false
+s3_default_bucket_location: ""
+s3_storage_container: ""
diff --git a/ansible/kibana-proxy-deploy.yml b/ansible/kibana-proxy-deploy.yml
deleted file mode 100644
index 7263492e85..0000000000
--- a/ansible/kibana-proxy-deploy.yml
+++ /dev/null
@@ -1,7 +0,0 @@
-- hosts: telemetry-search-cluster-kibana-node
-  become: yes
-  remote_user: ecosystem
-  roles:
-    - kibana-oauth-proxy
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
diff --git a/ansible/kibana_provision.yml b/ansible/kibana_provision.yml
deleted file mode 100644
index 8c9dba46f9..0000000000
--- a/ansible/kibana_provision.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-- hosts: telemetry-search-cluster-kibana-node
-  name: Setup Kibana
-  remote_user: ecosystem
-  become: yes
-  roles:
-    - kibana
diff --git a/ansible/lpa_api_provision.yml b/ansible/lpa_api_provision.yml
deleted file mode 100644
index f9377dc886..0000000000
--- a/ansible/lpa_api_provision.yml
+++ /dev/null
@@ -1,9 +0,0 @@
----
-- hosts: analytics-api
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
-  become: yes
-  gather_facts: false
-  roles:
-   - analytics-bootstrap-always       
-   - analytics-api-provision
diff --git a/ansible/lpa_cassandra_provision.yml b/ansible/lpa_cassandra_provision.yml
deleted file mode 100644
index 95df6f1c3d..0000000000
--- a/ansible/lpa_cassandra_provision.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-- hosts: cassandra 
-  # Install cassandra
-  become: yes
-  roles:
-    - analytics-bootstrap-always
-    - cassandra-3.7.0
diff --git a/ansible/lpa_data-products_deploy.yml b/ansible/lpa_data-products_deploy.yml
index baa1ab7227..7e84ba53f7 100644
--- a/ansible/lpa_data-products_deploy.yml
+++ b/ansible/lpa_data-products_deploy.yml
@@ -4,5 +4,8 @@
     - "{{inventory_dir}}/secrets.yml"
   become: yes
   become_user: "{{ analytics_user }}"
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
+    AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
   roles:
    - data-products-deploy
diff --git a/ansible/monit-deploy.yml b/ansible/monit-deploy.yml
index 69808b3667..c851142051 100644
--- a/ansible/monit-deploy.yml
+++ b/ansible/monit-deploy.yml
@@ -31,7 +31,7 @@
     - ['{{inventory_dir}}/secrets.yml']
   name: Installing and configuring monit
   roles:
-    - { role: monit, service: 'monit_assess-events-backup,monit_channel-telemetry-backup,monit_denorm-events-backup,monit_derived-telemetry-backup,monit_device-profile-backup,monit_extractor-failed-backup,monit_failed-telemetry-backup,monit_ingestion-telemetry-backup,monit_learning-events-backup,monit_learning-failed-backup,monit_pipeline-metrics-backup,monit_raw-telemetry-backup,monit_unique-telemetry-backup' }
+    - { role: monit, service: 'monit_assess-events-backup,monit_denorm-events-backup,monit_derived-telemetry-backup,monit_device-profile-backup,monit_extractor-failed-backup,monit_failed-telemetry-backup,monit_ingestion-cluster-telemetry-backup,monit_learning-events-backup,monit_learning-failed-backup,monit_raw-telemetry-backup,monit_unique-telemetry-backup,monit_assess-raw-events-backup,monit_telemetry-duplicate-backup,monit_extractor-duplicate-backup,monit_content-consumption-events-backup,monit_derived-denorm-events-backup,monit_channel-summary-backup' }
   tags: secor
 
 - hosts: kafka-indexer:analytics-api
diff --git a/ansible/portal.yml b/ansible/portal.yml
index 6a25409041..02c080c034 100644
--- a/ansible/portal.yml
+++ b/ansible/portal.yml
@@ -5,4 +5,4 @@
   roles:
     - portal-dashboard
   environment:
-    API_KEY: "{{dp_search_service_authorization_token}}"
\ No newline at end of file
+    API_KEY: "{{ sunbird_api_auth_token }}"
diff --git a/ansible/postgres-db-update.yml b/ansible/postgres-db-update.yml
new file mode 100644
index 0000000000..a3fee0c9f7
--- /dev/null
+++ b/ansible/postgres-db-update.yml
@@ -0,0 +1,7 @@
+---
+- hosts: local
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  become: yes
+  roles:
+    - postgres-db-update
diff --git a/ansible/postgres-manage.yml b/ansible/postgres-manage.yml
deleted file mode 100644
index e6abda59c2..0000000000
--- a/ansible/postgres-manage.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-- hosts: local
-  vars_files:
-    - "{{inventory_dir}}/secrets.yml"
-  become: yes
-  roles:
-    - postgres
diff --git a/ansible/postgres-managed-service-backup.yml b/ansible/postgres-managed-service-backup.yml
index 2fa88e3557..fada65ec35 100644
--- a/ansible/postgres-managed-service-backup.yml
+++ b/ansible/postgres-managed-service-backup.yml
@@ -4,6 +4,6 @@
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
   roles:
-    - postgres-azure-managed-service
+    - postgres-managed-service
   tags:
-    - postgres-azure-managed-service
\ No newline at end of file
+    - postgres-managed-service
diff --git a/ansible/postgresql-restore.yml b/ansible/postgresql-restore.yml
index e2d80770d1..bcec7447d6 100644
--- a/ansible/postgresql-restore.yml
+++ b/ansible/postgresql-restore.yml
@@ -3,6 +3,6 @@
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
   roles:
-    - postgres-azure-managed-service-restore
+    - postgres-managed-service-restore
   tags:
     - postgresql-restore
diff --git a/ansible/push_logs_azure.yml b/ansible/push_logs_azure.yml
deleted file mode 100644
index 887ec794ac..0000000000
--- a/ansible/push_logs_azure.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-- hosts: yarn-slave
-  become: yes
-  tasks:
-  - name: Copy the script
-    copy: src=resources/copy_azure.sh dest="{{script_path}}" owner=hduser group=hadoop mode="u=rwx,g=rx,o=r"
diff --git a/ansible/redis-backup.yml b/ansible/redis-backup.yml
index 248367c3fb..a767d32b00 100644
--- a/ansible/redis-backup.yml
+++ b/ansible/redis-backup.yml
@@ -1,10 +1,7 @@
-- hosts: redisall-backup
+- hosts: redisall
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
   roles:
-   - redis-backup
+   - redis-multiprocess-backup
   run_once: true
diff --git a/ansible/redis-restore.yml b/ansible/redis-restore.yml
index 69cc454f99..44c96511b0 100644
--- a/ansible/redis-restore.yml
+++ b/ansible/redis-restore.yml
@@ -6,4 +6,4 @@
   vars_files:
    - ['{{inventory_dir}}/secrets.yml', 'secrets/{{env}}.yml']
   roles:
-   - redis-restore
+   - redis-multiprocess-restore
diff --git a/ansible/redis_provision.yml b/ansible/redis_provision.yml
index 0efa7b9365..3bd750765e 100644
--- a/ansible/redis_provision.yml
+++ b/ansible/redis_provision.yml
@@ -16,6 +16,29 @@
         groups: sudo
         shell: /bin/bash
   roles:
-    - redis
+    - redis-multiprocess
   serial: 1
+  tags:
+    - metadata_redis  # this tag installs multiple redis process in single server
 
+- hosts: redis
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  pre_tasks:
+    - name: Create group
+      group:
+        name: analytics
+        state: present
+    - name: Create user
+      user:
+        name: analytics
+        comment: "analytics"
+        group: analytics
+        groups: sudo
+        shell: /bin/bash
+  roles:
+    - redis
+  serial: 1
+  tags: 
+    - single_redis_process # this tags installs single redis process.
diff --git a/ansible/roles/analytics-api-deploy/defaults/main.yml b/ansible/roles/analytics-api-deploy/defaults/main.yml
deleted file mode 100644
index e32185b9b6..0000000000
--- a/ansible/roles/analytics-api-deploy/defaults/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-kafka_broker: "{{groups['processing-cluster-kafka'][0]}}:9092"
-
-analytics:
-   home: /mount/data/analytics
-analytics_api:
-   root: "{{ analytics.home }}/api"
-   home: "{{ analytics.home }}/api/analytics-api-2.0"
-   service_args: -Dconfig.file=/mount/data/analytics/api/{{ env }}.conf
-   paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data']
-cassandra:
-  keyspace_prefix: "{{ env }}_"
-
-geo_location_city: "{{ env }}_geo_location_city"
-geo_location_city_ipv4: "{{ env }}_geo_location_city_ipv4"
-report_config: "{{env}}_report_config"
-default_channel_id: "in.ekstep"
-consumer_channel_cache_type: ConsumerChannel
-artifact_name: "analytics-api-2.0-dist.zip"
-
-analytics: 
-   home: /mount/data/analytics
diff --git a/ansible/roles/analytics-api-deploy/tasks/main.yml b/ansible/roles/analytics-api-deploy/tasks/main.yml
deleted file mode 100644
index 52805ce030..0000000000
--- a/ansible/roles/analytics-api-deploy/tasks/main.yml
+++ /dev/null
@@ -1,91 +0,0 @@
-- name: checking the list of installed services
-  service_facts:
-
-- name: Stop the Monit service
-  service:
-    name: monit
-    state: stopped
-  when: ansible_facts.services.monit is defined
-
-- name: Status of the api service
-  command: "{{ sbin_path }}/api-service status"
-  become_user: "{{analytics_user}}"   
-  register: api_status
-
-- name: Stop the api service
-  command: "{{ sbin_path }}/api-service stop"
-  become_user: "{{analytics_user}}"  
-  when: "api_status.stdout.find('RUNNING') != -1"
-  async: 10
-  poll: 5
-
-- name: Delete the pid file
-  file: path={{ sbin_path }}/api-service.pid state=absent
-  become_user: "{{analytics_user}}"
-
-- name: Delete the play pid file
-  file: path={{ analytics_api.home }}/RUNNING_PID state=absent
-  become_user: "{{analytics_user}}"
-
-- name: Remove lib directory
-  file:
-    path: "{{ analytics_api.home }}/lib"
-    state: absent
-  become_user: "{{analytics_user}}"    
-
-- name: Remove the old artifacts
-  file:
-    path: "{{ analytics_api.root }}/{{ artifact_name }}"
-    state: absent
-  become_user: "{{analytics_user}}"    
-      
-- name: Copy API artifact
-  copy: 
-    src: "{{ artifact_name }}"
-    dest: "{{ analytics_api.root }}"
-  become_user: "{{analytics_user}}"
-
-- name: Unarchive api artifact
-  unarchive: src={{ analytics_api.root }}/{{ artifact_name }} dest={{ analytics_api.root }} copy=no group={{ analytics_user }} owner={{ analytics_user }}
-  become_user: "{{analytics_user}}"
-    
-- name: Delete slf4j-log4j jar file
-  file: path={{ analytics_api.home }}/lib/org.slf4j.slf4j-log4j12-1.7.16.jar state=absent
-  become_user: "{{analytics_user}}"  
-
-- name: Copy API configuration file
-  template: src=conf.j2 dest={{ analytics_api.root }}/{{ env }}.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }}
-  become_user: "{{analytics_user}}"
-
-- name: Remove netty-all 4.0.29 from lib folder
-  file: path={{ analytics_api.home }}/lib/io.netty.netty-all-4.0.29.Final.jar state=absent
-  become_user: "{{analytics_user}}"
-
-- name: Start the api service
-  command: "{{ sbin_path }}/api-service start"
-  become_user: "{{analytics_user}}"   
-  async: 600
-  poll: 120
-
-- name: Check the API health status
-  uri:
-    url: http://localhost:9000/health
-  become_user: "{{analytics_user}}"    
-  register: api_health_status
-
-- name: Print the health status to console
-  debug:
-    var: api_health_status
-
-#Update ConsumerChannel Cache
-- uri:
-    url: "http://localhost:9000/refresh-cache/{{ consumer_channel_cache_type }}"
-    method: GET
-    return_content: yes
-  tags: refresh-cache
-
-- name: Start the Monit service
-  service:
-    name: monit
-    state: started
-  when: ansible_facts.services.monit is defined
diff --git a/ansible/roles/analytics-api-deploy/templates/conf.j2 b/ansible/roles/analytics-api-deploy/templates/conf.j2
deleted file mode 100644
index 22afa19538..0000000000
--- a/ansible/roles/analytics-api-deploy/templates/conf.j2
+++ /dev/null
@@ -1,257 +0,0 @@
-application.env="{{ env }}"
-spark.cassandra.connection.host={{groups['dp-cassandra'][0]}}
-spark.cassandra.connection.port="9042"
-service.search.url="{{lp_search}}"
-service.search.path="/v2/search"
-service.search.requestbody="""{"request":{"filters":{"objectType":["Content"],"contentType":["Story","Worksheet","Collection","Game"],"status":["Live"]},"limit":1000}}"""
-service.search.limit="10"
-
-metrics.creation.es.url="{{lp_composite_search}}"
-metrics.creation.es.indexes="compositesearch"
-
-
-recommendation.enable=true
-recommendation.limit="10"
-recommendation.surprise_find.enable=true
-recommendation.surprise_find.index="15"
-recommendation.surprise_find.serendipity_factor="20"
-
-dataproduct.scripts_path="/mount/data/analytics/scripts"
-
-# Metrics API
-# metrics.period.format.day="MMM dd EEE"
-# metrics.period.format.month="MMM YYYY"
-# metrics.period.format.year="YYYY"
-
-
-# Data Exhaust API
-data_exhaust.list.limit="100"
-data_exhaust.retry.limit="3"
-data_exhaust.dataset.list=["eks-consumption-raw", "eks-consumption-summary", "eks-consumption-metrics","eks-creation-raw", "eks-creation-summary", "eks-creation-metrics"]
-data_exhaust.dataset.default="eks-consumption-raw"
-data_exhaust.output_format="json"
-
-# Log4j Kafka appender config
-log4j.appender.kafka.enable="false"
-log4j.appender.kafka.broker_host="{{ kafka_broker }}"
-log4j.appender.kafka.topic="{{env}}.telemetry.log"
-
-cassandra.service.embedded.enable=false
-cassandra.keyspace_prefix="{{ cassandra.keyspace_prefix }}"
-
-device-register-controller-dispatcher {
-  type = "Dispatcher"
-  executor = "fork-join-executor"
-  fork-join-executor {
-    # The parallelism factor is used to determine thread pool size using the
-    # following formula: ceil(available processors * factor). Resulting size
-    # is then bounded by the parallelism-min and parallelism-max values.
-    parallelism-factor = 3.0
-
-    # Min number of threads to cap factor-based parallelism number to
-    parallelism-min = 4
-
-    # Max number of threads to cap factor-based parallelism number to
-    parallelism-max = 8
-  }
-  # Throughput for default Dispatcher, set to 1 for as fair as possible
-  throughput = 1
-}
-
-device-register-actor-dispatcher {
-  type = "Dispatcher"
-  executor = "thread-pool-executor"
-  thread-pool-executor {
-    fixed-pool-size = 8
-  }
-  throughput = 1
-}
-
-device-profile-actor-dispatcher {
-  type = "Dispatcher"
-  executor = "thread-pool-executor"
-  thread-pool-executor {
-    fixed-pool-size = 8
-  }
-  throughput = 1
-}
-
-experiment-actor {
-  type = "Dispatcher"
-  executor = "fork-join-executor"
-  fork-join-executor {
-    # The parallelism factor is used to determine thread pool size using the
-    # following formula: ceil(available processors * factor). Resulting size
-    # is then bounded by the parallelism-min and parallelism-max values.
-    parallelism-factor = 3.0
-
-    # Min number of threads to cap factor-based parallelism number to
-    parallelism-min = 8
-
-    # Max number of threads to cap factor-based parallelism number to
-    parallelism-max = 16
-  }
-  # Throughput for default Dispatcher, set to 1 for as fair as possible
-  throughput = 1
-}
-
-default-dispatcher {
-  executor = "fork-join-executor"
-  fork-join-executor {
-    # The parallelism factor is used to determine thread pool size using the
-    # following formula: ceil(available processors * factor). Resulting size
-    # is then bounded by the parallelism-min and parallelism-max values.
-    parallelism-factor = 3.0
-
-    # Min number of threads to cap factor-based parallelism number to
-    parallelism-min = 8
-
-    # Max number of threads to cap factor-based parallelism number to
-    parallelism-max = 16
-  }
-  # Throughput for default Dispatcher, set to 1 for as fair as possible
-  throughput = 1
-}
-
-#AKKA Configuration
-akka {
-  actor {
-    deployment {
-
-        /druid-health-actor  {
-            router = smallest-mailbox-pool
-            nr-of-instances = 10
-        }
-        /job-service-actor {
-          router = smallest-mailbox-pool
-            nr-of-instances = 10
-        }
-        /device-register-actor {
-          router = smallest-mailbox-pool
-          dispatcher = device-register-actor-dispatcher
-          nr-of-instances = 6
-        }
-        /device-profile-actor {
-          router = smallest-mailbox-pool
-          dispatcher = device-profile-actor-dispatcher
-          nr-of-instances = 6
-        }
-        /client-log-actor {
-          router = smallest-mailbox-pool
-          nr-of-instances = 4
-        }
-        /experiment-actor {
-          router = smallest-mailbox-pool
-          nr-of-instances = 2
-        }
-
-    }
-  }
-}
-
-#Netty Configuration
-play.server {
-
-  # The server provider class name
-  provider = "play.core.server.NettyServerProvider"
-
-  netty {
-
-    # The number of event loop threads. 0 means let Netty decide, which by default will select 2 times the number of
-    # available processors.
-    eventLoopThreads = 30
-
-    # Whether the Netty wire should be logged
-    log.wire = true
-
-    # The transport to use, either jdk or native.
-    # Native socket transport has higher performance and produces less garbage but are only available on linux
-    transport = "native"
-  }
-}
-
-play.modules.enabled+="modules.ActorInjector"
-play.filters.enabled+="filter.RequestInterceptor"
-play.filters.enabled+="play.filters.hosts.AllowedHostsFilter"
-play.filters.disabled+="play.filters.csrf.CSRFFilter"
-
-# Secret key
-# ~~~~~
-# The secret key is used to secure cryptographics functions.
-# If you deploy your application to several instances be sure to use the same key!
-play.http.secret.key="{{dp_play_http_secret_key}}"
-
-play.filters.hosts {
-   # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
-   # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
-   # "." matches all domains, and "" matches an empty or nonexistent host.
-  allowed = ["."]
-}
-
-# body parser
-play.http.parser.maxMemoryBuffer=10M
-
-default.consumption.app.id="no_value"
-default.channel.id="{{default_channel_id}}"
-default.creation.app.id="no_value"
-
-postgres.db="{{postgres.db_name}}"
-postgres.url="jdbc:postgresql://{{ postgres.db_url }}:5432/"
-postgres.user="{{ postgres.db_username }}"
-postgres.pass="{{ postgres.db_password }}"
-postgres.table_name="{{ postgres.db_table_name }}"
-postgres.table.geo_location_city.name="{{ geo_location_city }}"
-postgres.table.geo_location_city_ipv4.name="{{ geo_location_city_ipv4 }}"
-postgres.table.report_config.name="{{ report_config }}"
-
-default.channel="{{ default_channel }}"
-
-elasticsearch.service.endpoint="http://{{groups['composite-search-cluster'][0]}}:9200"
-elasticsearch.index.compositesearch.name="compositesearch"
-elasticsearch.index.dialcodemetrics.name="dialcodemetrics"
-metrics.dialcodemetrics.request.limit=1000
-
-#channel specifix telemetry exhaust
-
-dataexhaust.authorization_check=true
-
-channel.data_exhaust.bucket="{{channel_data_exhaust_bucket}}"
-channel.data_exhaust.basePrefix="channel/"
-channel.data_exhaust.expiryMins=30
-storage-service.request-signature-version="AWS4-HMAC-SHA256"
-s3service.region="ap-south-1"
-
-cloud_storage_type="azure"
-
-metrics.time.interval.min=30
-cache.refresh.time.interval.min=1440
-
-redis.host="{{metadata_redis_host}}"
-#redis.host="localhost"
-redis.port=6379
-#redis.port=__redis_port__
-redis.connection.max=20
-redis.connection.idle.max=20
-redis.connection.idle.min=10
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-
-elasticsearch.host="{{lp_composite_search_host}}"
-elasticsearch.port=9200
-
-# experiment service settings
-elasticsearch.searchExperiment.index="experiment"
-elasticsearch.searchExperiment.fieldWeight="{\"userId\":3.0, \"deviceId\":3.0, \"url\":3.0 }"
-elasticsearch.searchExperiment.matchQueryScore=9.0
-deviceRegisterAPI.experiment.enable=false
-experimentService.redisEmptyValueExpirySeconds=86400
-redis.experimentIndex=9
-redis.deviceIndex=2
-
-druid.coordinator.host="http://{{groups['raw-coordinator'][0]}}:8081/"
-druid.healthcheck.url="druid/coordinator/v1/loadstatus"
-
-device.api.enable.debug.log=true
-kafka.device.register.topic={{ env }}.events.deviceprofile
-kafka.metrics.event.topic={{ env }}.pipeline_metrics
-kafka.broker.list="{{ ingestion_kafka_brokers }}"
diff --git a/ansible/roles/analytics-api-provision/defaults/main.yml b/ansible/roles/analytics-api-provision/defaults/main.yml
deleted file mode 100644
index 36875b3a47..0000000000
--- a/ansible/roles/analytics-api-provision/defaults/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-analytics_user_home: /home/{{ analytics_user }}
-sbin_path: "{{ analytics_user_home }}/sbin"
-storage_provider_env: /etc/environment
-script_name: api-service
-script_dir: "{{ sbin_path }}"
-description: Analytics API service
-grep_key: play.core.server.ProdServerStart
-process: "/usr/bin/java"
-analytics_api_min_heap: 1g
-analytics_api_max_heap: 2g
-java_options: "-XX:+UseG1GC -XX:+UseStringDeduplication"
-process_home: ""
-process_args: -- -cp '{{ analytics_api.home }}/lib/*' {{ analytics_api.service_args }} play.core.server.ProdServerStart {{ analytics_api.home }}
-analytics_api_version: "2.0"
-app_home: "/mount/data/analytics/api/analytics-api-{{ analytics_api_version }}"
-
-
-analytics_api:
-   root: "{{analytics.home}}/api"
-   home: "{{ analytics.home }}/api/analytics-api-{{ analytics_api_version }}"
-   service_args: -Dconfig.file=/mount/data/analytics/api/{{ env }}.conf -Xms{{analytics_api_min_heap}} -Xmx{{analytics_api_max_heap}} {{java_options}}
-   paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data']
-
-analytics: 
-   home: /mount/data/analytics
diff --git a/ansible/roles/analytics-api-provision/tasks/main.yml b/ansible/roles/analytics-api-provision/tasks/main.yml
deleted file mode 100644
index 87a666ef8a..0000000000
--- a/ansible/roles/analytics-api-provision/tasks/main.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-## Bootstrap for analytics-api ##
-
-- name: Adding ENV Vars to bashrc file of spark.
-  lineinfile:
-    path: '{{ analytics_user_home }}/.bashrc'
-    line: 'export {{item.var}}={{item.value}}'
-    regexp: "export {{ item.var }}.*"
-  with_items:
-    - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
-    - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
-
-- name: Adding ENV Vars to spark servers environment.
-  lineinfile:
-    path: '/etc/environment'
-    line: '{{item.var}}={{item.value}}'
-    regexp: "{{ item.var }}.*"
-  with_items:
-    - {var: 'azure_storage_key', value: '{{ sunbird_private_storage_account_name }}'}
-    - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
-
-- name: Install libraries for spark bootstrap
-  become: yes
-  apt:
-    name: "{{ item }}"
-    state: present
-    update_cache: yes
-  with_items:
-    - build-essential
-    - git
-
-- name: Create directories for analytics API
-  file:
-    path: "{{ item }}"
-    owner: "{{ analytics_user }}"
-    group: "{{ analytics_group }}"
-    state: directory
-  with_items: "{{ analytics_api.paths }}"
-
-- name: Setup analytics API as service
-  template:
-    src: servicify-process.j2
-    dest: "{{ sbin_path }}/api-service"
-    mode: 0755
-    owner: "{{ analytics_user }}"
-    group: "{{ analytics_group }}"
diff --git a/ansible/roles/analytics-api-provision/templates/servicify-process.j2 b/ansible/roles/analytics-api-provision/templates/servicify-process.j2
deleted file mode 100644
index a203423cef..0000000000
--- a/ansible/roles/analytics-api-provision/templates/servicify-process.j2
+++ /dev/null
@@ -1,104 +0,0 @@
-#!/bin/bash
-
-# chkconfig: 2345 95 05
-# description: {{ description }}
-
-# Source function library.
-. /lib/lsb/init-functions
-
-prog="{{ script_name }}"
-SCRIPT_DIR="{{ script_dir }}" #/home/ubuntu/
-PIDFILE={{ script_dir }}/$prog.pid
-DESC="{{ description }}"
-GREP_KEY="{{ grep_key }}"
-DAEMON={{ process }}
-DAEMON_HOME="{{ process_home }}"
-LOGS_HOME="{{ analytics.home }}/logs"
-APP_HOME="{{ app_home }}"
-
-
-#echo "All value sets"
-
-start() {
-    if [ -f $PIDFILE ]; then
-        PID=`cat $PIDFILE`
-        if [ -z "`pgrep $PID`" ] && [ "$PID" != "`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`" ]; then
-            printf "%s\n" "Process dead but pidfile exists"
-        else
-            printf "$prog is already running!\n"
-        fi
-    else
-        printf "%-50s" "Starting $prog ..."
-        export azure_storage_key="{{sunbird_private_storage_account_name}}"
-        export azure_storage_secret="{{sunbird_private_storage_account_key}}"
-        cd $APP_HOME
-        if [ -z "$DAEMON_HOME" ]; then
-            start-stop-daemon --start --quiet --background --no-close --name $prog --exec $DAEMON {{ process_args }} >$LOGS_HOME/services/$prog.log 2>&1
-        else 
-            start-stop-daemon --start --quiet --background -d $DAEMON_HOME --exec $DAEMON {{ process_args }} >$LOGS_HOME/services/$prog.log 2>&1
-        fi
-        sleep 10
-        PID=`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`
-        if [ -z "$PID" ]; then
-            printf "[ \e[31mFAIL\033[0m ]\n"
-            exit 1
-        else
-            echo $PID > $PIDFILE
-            printf "[ \e[32mOK\033[0m ]\n"
-        fi
-    fi
-}
-
-stop() {
-    printf "%-50s" "Shutting down $prog:"
-    if [ -f $PIDFILE ]; then
-        PID=`cat $PIDFILE`
-        start-stop-daemon --stop --quiet --oknodo --pidfile $PIDFILE 2>/dev/null
-        sleep 5
-        PID=`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`
-        if [ ! -z "$PID" ]; then
-            printf "[ \e[31mFAIL\033[0m ]\n"
-            exit 1
-        else
-            rm -f $PIDFILE
-            printf "[ \e[32mOK\033[0m ]\n"
-        fi
-    else
-        printf "[ \e[32mNOT RUNNING\033[0m ]\n"
-    fi
-}
-
-check_status() {
-    printf "%-50s" "Checking $prog ..."
-    if [ -f $PIDFILE ]; then
-        PID=`cat $PIDFILE`
-        if [ -z "`pgrep $PID`" ] && [ "$PID" != "`ps aux|grep -vE 'grep|runuser|bash'|grep -w "$GREP_KEY"|awk '{print $2}'`" ]; then
-            printf "%s\n" "Process dead but pidfile exists"
-        else
-            printf "[ \e[32mRUNNING\033[0m ]\n"
-        fi
-    else
-        printf "[ \e[31mSTOPPED\033[0m ]\n"
-    fi
-}
-
-case "$1" in
-    start)
-        start
-        ;;
-    stop)
-        stop
-        ;;
-    status)
-        check_status
-        ;;
-    restart)
-        stop
-        start
-        ;;
-    *)
-        echo "Usage: $prog {start|stop|status|restart}"
-        exit 1
-        ;;
-esac
-exit 0
diff --git a/ansible/roles/analytics-bootstrap-always/meta/main.yml b/ansible/roles/analytics-bootstrap-always/meta/main.yml
index ce1479d7c0..af15826aef 100644
--- a/ansible/roles/analytics-bootstrap-always/meta/main.yml
+++ b/ansible/roles/analytics-bootstrap-always/meta/main.yml
@@ -1,6 +1,6 @@
 ---
 dependencies:
-  - { role: jdk-1.8.0_121 , become: yes }
+  - { role: jdk11 , become: yes }
   - { role: azure-cli  , become: yes }
     
   
diff --git a/ansible/roles/analytics-bootstrap-always/tasks/main.yml b/ansible/roles/analytics-bootstrap-always/tasks/main.yml
index d7e1594440..065a3b3afb 100644
--- a/ansible/roles/analytics-bootstrap-always/tasks/main.yml
+++ b/ansible/roles/analytics-bootstrap-always/tasks/main.yml
@@ -17,6 +17,7 @@
     createhome: yes
     group: "{{ analytics_group }}"
 
+
 - name: Installing packages
   become: yes
   apt:
@@ -26,7 +27,20 @@
   with_items:
     - rsync
     - zip
-    - python-pip
+
+- name: install python-pip
+  become: yes
+  apt:
+    state: present
+    name: python-pip
+  when: ansible_distribution_version | float < 18
+
+- name: install python-pip
+  become: yes
+  apt:
+    state: present
+    name: python3-pip
+  when: ansible_distribution_version | float > 18
 
 - name: Set locale LC_ALL=en_US.UTF-8
   become: yes
diff --git a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
index 8af0e60754..13ba75f78a 100644
--- a/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
+++ b/ansible/roles/analytics-bootstrap-spark/tasks/main.yml
@@ -22,6 +22,11 @@
     - {var: 'azure_storage_secret', value: '{{ sunbird_private_storage_account_key }}'}
     - {var: 'AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_private_storage_account_name }}'}
     - {var: 'AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_private_storage_account_key }}'}
+    - {var: 'PUBLIC_AZURE_STORAGE_ACCOUNT', value: '{{ sunbird_public_storage_account_name }}'}
+    - {var: 'PUBLIC_AZURE_STORAGE_ACCESS_KEY', value: '{{ sunbird_public_storage_account_key }}'}
+    - {var: 'PRIVATE_REPORT_CONTAINER', value: '{{ sunbird_private_azure_report_container_name }}'}
+    - {var: 'PUBLIC_REPORT_CONTAINER', value: '{{ sunbird_public_azure_report_container_name }}'}
+    - {var: 'REPORT_BACKUP_CONTAINER', value: 'portal-reports-backup'}
     - {var: 'GOOGLE_CREDENTIALS_PATH', value: '/home/analytics/credentials'}
     - {var: 'STORAGE_PROVIDER', value: 'AZURE'}
     - {var: 'ENV', value: '{{env}}'}
diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml
index 6a3d80e197..8a55bc3f97 100644
--- a/ansible/roles/analytics-druid/defaults/main.yml
+++ b/ansible/roles/analytics-druid/defaults/main.yml
@@ -1,25 +1,25 @@
 ---
 # Druid Template variables
-
-zookeeper_group: "{{ groups['druid-zookeeper'] }}"
+druid_monitoring: true
+zookeeper_group: "groups[{{ cluster }}-zookeeper]"
 
 druid_user: druid
 druid_directory: "/data"
-druid_version: "0.18.0"
-druid_url: "https://downloads.apache.org/druid/0.18.0/apache-druid-0.18.0-bin.tar.gz"
+druid_version: "0.22.1"
+druid_url: "https://archive.apache.org/dist/druid/{{druid_version}}/apache-druid-{{druid_version}}-bin.tar.gz"
 druid_folder: "{{ druid_directory }}/apache-druid-{{ druid_version }}"
 druid_symlink: "{{ druid_directory }}/druid"
 druid_path: "{{ druid_symlink }}/"
-druid_checksum: "807581d54fa4c5a90eec2a230e2a7fc4c6daf18eb8136009bf36a775d793d6f6"
+druid_checksum: "716b83e07a76b5c9e0e26dd49028ca088bde81befb070989b41e71f0e8082d11a26601f4ac1e646bf099a4bc7420bdfeb9f7450d6da53d2a6de301e08c3cab0d"
 
+druid_open_file_limit: 65535
+druid_process_limit: 128596
 #Common Varibles
 
-druid_zookeeper_host: "{{ groups['druid-zookeeper']|join(':2181,')}}:2181"
+druid_zookeeper_host: "{{ groups[cluster+'-zookeeper']|join(':2181,')}}:2181"
+druid_postgres_user: druid
 #druid_postgres_host: "{{ groups['druid-postgres'][0] }}"
-druid_postgres_host: "{{ postgres.db_url }}"
-druid_postgres_port: "{{ postgres.db_port }}"
-druid_postgres_user: "druid@{{ postgres.db_url }}"
-druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+
 
 druid_default_tmp_dir: "/var/tmp"
 druid_gc_logdir: "/var/log/druid/crash_logs"
@@ -29,23 +29,11 @@ druid_log_dir: "/var/log/druid/"
 #Writing request query logs to file 
 druid_request_logging_type: "file"
 
-#azure config in group vars
-
-druid_azure_container_name: "telemetry-data-store"
-azure_account_name: "{{sunbird_private_storage_account_name}}"
-azure_storage_secret: "{{sunbird_private_storage_account_key}}"
-azure_container: "{{ druid_azure_container_name }}"
-
-#Logging the indexing logs to azure
-
-druid_log_azure_container: "{{ druid_azure_container_name }}"
-druid_log_azure_folder: "druidlogs"
-
 #Druid Extensions
 
-druid_storge_type: "azure"
+druid_storage_type: "azure"
 
-druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service"'
+druid_extensions_list : '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"'
 
 druid_community_extensions:
   - graphite-emitter
@@ -56,16 +44,12 @@ druid_indexing_logs_type: azure
 druid_indexing_log_dir: /var/druid/indexing-logs
 druid_indexing_storage_type : metadata
 druid_indexing_task_basedir : "/var/task"
-druid_indexing_queue_startDelay: PT30S
 
 druid_common_monitors: '"com.metamx.metrics.JvmMonitor","org.apache.druid.java.util.metrics.JvmMonitor"'
 druid_common_emitters: '"logging","graphite"'
 
-druid_graphite_host: "{{ groups['druid-graphite'][0] }}"
-druid_graphite_port : 2004
 druid_graphite_prefix: "druid.metric"
 druid_whitelist_filepath: "{{ druid_path }}whitelist"
-druid_postgress_graphite_pass: "{{ dp_vault_druid_postgress_graphite_pass }}"
 
 #End of Common variables
 
@@ -74,9 +58,6 @@ druid_postgress_graphite_pass: "{{ dp_vault_druid_postgress_graphite_pass }}"
 druid_coordinator_port: 8081
 druid_coordinator_service : druid/coordinator
 
-druid_coordinator_heap_size: 128m
-druid_coordinator_startDelay: PT30S
-druid_coordinator_period: PT30S
 druid_coordinator_tmp_dir: "{{ druid_default_tmp_dir }}"
 druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.log"
 
@@ -92,25 +73,16 @@ druid_overlord_gc_logfile: "{{ druid_crash_logdir }}/gc_overlord.log"
 druid_broker_port: 8082
 druid_broker_service: druid/broker
 
-druid_broker_heap_size: 128m
 druid_broker_max_direct_size: 800m
 druid_broker_tmp_dir: /var/tmp
 druid_broker_gc_logdir: "{{ druid_crash_logdir }}/gc.log"
 
-druid_broker_http_numConnections: 5
-druid_broker_server_http_numThread: 25
-
-druid_broker_processing_bufferBytes: 134217728
-druid_broker_processing_threads: 2
-
-
 # Historical Configurations
 
 druid_historical_port: 8083
 druid_historical_service: druid/historical
 
-druid_historical_heap_size: 1048m
-druid_historical_max_direct_size: 800m
+
 druid_historical_tmp_dir: "{{ druid_default_tmp_dir }}"
 druid_historical_gc_logfile: "{{ druid_crash_logdir }}/historical.gc.log"
 druid_historical_heap_dump_file: "{{ druid_crash_logdir }}/historical.hprof"
@@ -119,22 +91,9 @@ druid_coordinator_heap_dump_file : "{{ druid_crash_logdir }}/coordinator.hprof"
 druid_overlord_heap_dump_file : "{{ druid_crash_logdir }}/overlord.hprof"
 druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof"
 
-druid_historical_http_numConnections: 5
-druid_historical_server_http_numThread: 25
-
-druid_historical_processing_bufferBytes: 134217728
-druid_historical_processing_threads: 2
-druid_historical_processing_num_merge_buffers: 2
-
-
 druid_historical_monitoring_monitors: '"org.apache.druid.server.metrics.HistoricalMetricsMonitor","com.metamx.metrics.JvmMonitor"'
 
 druid_historical_segmentcache_path: "/var/segmentstore"
-druid_historical_segmentcache_size: 10000000000
-
-druid_historical_server_maxsize: 10000000000
-
-druid_historical_enable_cache: false
 druid_historical_cache_size: 3221225472
 druid_historical_cache_expiry: 3600000
 
@@ -143,26 +102,166 @@ druid_historical_cache_expiry: 3600000
 druid_middlemanager_port: 8091
 druid_middlemanager_service: druid/middlemanager
 
-druid_middlemanager_heap_size: 128m
 druid_middlemanager_tmp_dir: "{{ druid_default_tmp_dir }}"
 druid_middlemanager_gc_logdir: "{{ druid_crash_logdir }}/gc.log"
 
-druid_middlemanager_worker_cap: 4
-druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
-
-druid_middlemanager_peon_server_http_numThread: 25
-druid_middlemanager_peon_processing_bufferBytes: 25000000
-druid_middlemanager_peon_processing_threads: 2
-
-
 druid_middlemanager_peon_segmentcache_path: "/var/segment_cache"
-druid_middlemanager_peon_server_maxsize: 0
-
-# Router configurations	
+druid_azure_container_name: "telemetry-data-store"
 
-druid_router_heap_size: 1g
-druid_router_http_numConnections: 50
-druid_router_http_readTimeout: PT5M
-druid_router_http_numMaxThreads: 100
-druid_server_http_numThreads: 100
-druid_router_managementProxy_enabled: true
+# Router Configuration
+druid_router_port: 8888
+druid_router_service: druid/router
+
+druid_router_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_router_gc_logfile: "{{ druid_crash_logdir }}/gc.router.log"
+druid_router_heap_dump_file: "{{ druid_crash_logdir }}/router.hprof"
+
+
+default_druid_configs:
+  raw:
+   #Druid Postgres Details
+   druid_postgres_db: "druid"
+   druid_postgres_host: "{{ postgres.db_url }}"
+   druid_postgres_port: "{{ postgres.db_port }}"
+   druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
+   #Druid Azure Details 
+   druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
+   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
+   azure_container: "{{ druid_azure_container_name }}"
+   #Logging the indexing logs to azure
+   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_folder: "druidlogs"
+   #Druid S3 Details
+   druid_storage_type: "{{ druid_storage_type }}"
+   s3_access_key: "{{ s3_storage_key }}"
+   s3_secret_key: "{{ s3_storage_secret }}"
+   s3_bucket: "{{ s3_storage_container }}"
+   s3_endpoint: "{{ s3_storage_endpoint }}"
+   s3_segment_dir: "druid/raw/segments"
+   s3_path_like_access: "{{ s3_path_style_access }}"
+   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   #Logging the indexing logs to s3
+   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_indexer_logs_dir: "druid/raw/stage/indexing_logs"
+   #Druid coordinator node configuration
+   druid_coordinator_heap_size: 128m
+   druid_coordinator_period: PT30S
+   druid_coordinator_startDelay: PT30S
+   druid_coordinator_balance_strategy: diskNormalized
+   #Druid overlord node configuration
+   druid_overlord_heap_size: 256m
+   #Druid broker node configuration
+   druid_broker_min_heap_size: 128m
+   druid_broker_max_heap_size: 128m
+   druid_broker_max_direct_size: 800m
+   druid_broker_http_numConnections: 5
+   druid_broker_server_http_numThread: 25
+   druid_broker_processing_bufferBytes: 134217728
+   druid_broker_processing_threads: 2
+   #Druid historical node configuration
+   druid_historical_min_heap_size: 1048m
+   druid_historical_max_heap_size: 1048m
+   druid_historical_max_direct_size: 800m
+   druid_historical_http_numConnections: 5
+   druid_historical_server_http_numThread: 25
+   druid_historical_processing_bufferBytes: 134217728
+   druid_historical_processing_threads: 2
+   druid_historical_enable_cache: false
+   druid_historical_segmentcache_size: 10000000000
+   druid_historical_server_maxsize: 10000000000
+   druid_historical_processing_num_merge_buffers: 2
+   druid_query_ondiskstorage_enabled: true
+   druid_historical_maxMergingDictionarySize: 100000000
+   druid_historical_segmentcache_numloadingthreads: 4
+   druid_historical_segmentcache_path: "/var/segmentstore"
+   druid.query.groupBy.maxOnDiskStorage: 10737418240
+   #Druid middlemanager configuration
+   druid_middlemanager_heap_size: 128m
+   druid_middlemanager_worker_cap: 4
+   druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+   druid_middlemanager_peon_server_http_numThread: 25
+   druid_middlemanager_peon_processing_bufferBytes: 25000000
+   druid_middlemanager_peon_processing_threads: 2
+   druid_middlemanager_peon_server_maxsize: 0
+   druid_indexing_queue_startDelay: PT30S
+   druid_router_heap_size: 1g
+   druid_router_http_numConnections: 50
+   druid_router_http_readTimeout: PT5M
+   druid_router_http_numMaxThreads: 100
+   druid_server_http_numThreads: 100
+   druid_router_managementProxy_enabled: true
+   druid_historical_maxOnDiskStorage: 10737418240
+  rollup:
+   #Druid Postgres Details
+   druid_postgres_db: "druid"
+   druid_postgres_host: "{{ postgres.db_url }}"
+   druid_postgres_port: "{{ postgres.db_port }}"
+   druid_postgres_user: "{{ druid_postgres_user | default('druid@' + postgres.db_url) }}"
+   #Druid Azure Details
+   druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
+   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
+   azure_container: "{{ druid_azure_container_name }}"
+   #Logging the indexing logs to azure
+   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_folder: "druidlogs"
+   #Druid S3 Details
+   druid_storage_type: "{{ druid_storage_type }}"
+   s3_access_key: "{{ s3_storage_key }}"
+   s3_secret_key: "{{ s3_storage_secret }}"
+   s3_bucket: "{{ s3_storage_container }}"
+   s3_endpoint: "{{ s3_storage_endpoint }}"
+   s3_segment_dir: "druid/rollup/segments"
+   s3_path_like_access: "{{ s3_path_style_access }}"
+   s3_v4_sign_region: "{{ s3_default_bucket_location }}"
+   #Logging the indexing logs to s3
+   s3_logging_bucket: "{{ s3_storage_container }}"
+   s3_indexer_logs_dir: "druid/rollup/stage/indexing_logs"
+   #Druid coordinator node configuration
+   druid_coordinator_heap_size: 128m
+   druid_coordinator_period: PT30S
+   druid_coordinator_startDelay: PT30S
+   druid_coordinator_balance_strategy: cost
+   #Druid overlord node configuration
+   druid_overlord_heap_size: 256m
+   #Druid broker node configuration
+   druid_broker_min_heap_size: 128m
+   druid_broker_max_heap_size: 128m
+   druid_broker_max_direct_size: 700m
+   druid_broker_http_numConnections: 5
+   druid_broker_server_http_numThread: 25
+   druid_broker_processing_bufferBytes: 134217728
+   druid_broker_processing_threads: 2
+   #Druid historical node configuration
+   druid_historical_min_heap_size: 1048m
+   druid_historical_max_heap_size: 1048m
+   druid_historical_max_direct_size: 800m
+   druid_historical_http_numConnections: 5
+   druid_historical_server_http_numThread: 25
+   druid_historical_processing_bufferBytes: 134217728
+   druid_historical_processing_threads: 2
+   druid_historical_enable_cache: false
+   druid_historical_segmentcache_size: 2000000000
+   druid_historical_server_maxsize: 10000000000
+   druid_historical_processing_num_merge_buffers: 2
+   druid_query_ondiskstorage_enabled: false
+   druid_historical_segmentcache_numloadingthreads: 4
+   druid_historical_segmentcache_path: "/var/segmentstore"
+   #Druid middlemanager configuration
+   druid_middlemanager_heap_size: 128m
+   druid_middlemanager_worker_cap: 4
+   druid_mm_java_opts_array: "-server -Xmx900m -XX:+UseG1GC -XX:HeapDumpPath={{ druid_mm_heap_dump_file }} -XX:MaxGCPauseMillis=100 -XX:+PrintGCDetails -XX:+PrintGCTimeStamps"
+   druid_middlemanager_peon_server_http_numThread: 25
+   druid_middlemanager_peon_processing_bufferBytes: 25000000
+   druid_middlemanager_peon_processing_threads: 2
+   druid_middlemanager_peon_server_maxsize: 0
+   druid_indexing_queue_startDelay: PT30S
+   druid_router_heap_size: 1g
+   druid_router_http_numConnections: 50
+   druid_router_http_readTimeout: PT5M
+   druid_router_http_numMaxThreads: 100
+   druid_server_http_numThreads: 100
+   druid_router_managementProxy_enabled: true
+
+enable_druid_sql: true
diff --git a/ansible/roles/analytics-druid/files/whitelist b/ansible/roles/analytics-druid/files/whitelist
index e96d9c8679..9ce2d7bf71 100644
--- a/ansible/roles/analytics-druid/files/whitelist
+++ b/ansible/roles/analytics-druid/files/whitelist
@@ -1,111 +1,11 @@
 {
-  "jetty/numOpenConnections" :[],
-  "query/time" : ["dataSource", "type"],    
+  "tier/historical/count":["tier"],
+  "segment/unavailable/count": ["dataSource"],
+  "task/success/count":["dataSource"],
+  "task/failed/count":["dataSource"],
+  "ingest/kafka/lag" : ["dataSource"],
+  "ingest/handoff/failed" : ["dataSource"],
   "query/success/count" : [],
   "query/failed/count" : [],
-  "query/interrupted/count" : [],
-  "query/node/time" : ["server"],
-  "query/node/ttfb" : ["server"],
-  "query/intervalChunk/time" : [],
-
-  "query/segment/time" :[],
-  "query/wait/time" : [],
-  "segment/scan/pending" : [],
-  "query/segmentAndCache/time" : [],
-  "query/cpu/time" : ["dataSource", "type"],
-
-  "query/cache/delta/numEntries" : [],
-  "query/cache/delta/sizeBytes" : [],
-  "query/cache/delta/hits" :  [],
-  "query/cache/delta/misses" : [],
-  "query/cache/delta/evictions" :  [],
-  "query/cache/delta/hitRate" : [],
-  "query/cache/delta/averageBytes" : [],
-  "query/cache/delta/timeouts" :  [],
-  "query/cache/delta/errors" :  [],
-
-  "query/cache/total/numEntries" : [],
-  "query/cache/total/sizeBytes" : [],
-  "query/cache/total/hits" :  [],
-  "query/cache/total/misses" : [],
-  "query/cache/total/evictions" :  [],
-  "query/cache/total/hitRate" : [],
-  "query/cache/total/averageBytes" : [],
-  "query/cache/total/timeouts" :  [],
-  "query/cache/total/errors" : [],
-
-  "ingest/events/thrownAway" :  ["dataSource", "taskType"],
-  "ingest/events/unparseable" : ["dataSource", "taskType"],
-  "ingest/events/processed" :  ["dataSource", "taskType"],
-  "ingest/rows/output" : ["dataSource"],
-  "ingest/persist/count" : ["dataSource"],
-  "ingest/persist/time" : ["dataSource"],
-  "ingest/persist/cpu" : ["dataSource"],
-  "ingest/persist/backPressure" : ["dataSource"],
-  "ingest/persist/failed" : ["dataSource"],
-  "ingest/handoff/failed" : ["dataSource"],
-  "ingest/merge/time" : ["dataSource"],
-  "ingest/merge/cpu" : ["dataSource"],
-
-  "task/run/time" :  ["dataSource", "taskType"],
-  "segment/added/bytes" :  ["dataSource", "taskType"],
-  "segment/moved/bytes" :  ["dataSource", "taskType"],
-  "segment/nuked/bytes" :  ["dataSource", "taskType"],
-
-  "segment/assigned/count" : ["tier"],
-  "segment/moved/count" : ["tier"],
-  "segment/dropped/count" : ["tier"],
-  "segment/deleted/count" : ["tier"],
-  "segment/unneeded/count" : ["tier"],
-  "segment/cost/raw" : ["tier"],
-  "segment/cost/normalization" : ["tier"],
-  "segment/cost/normalized" : ["tier"],
-  "segment/loadQueue/size" : ["server"],
-  "segment/loadQueue/failed" : ["server"],
-  "segment/loadQueue/count" : ["server"],
-  "segment/dropQueue/count" : ["server"],
-  "segment/size" : ["dataSource"],
-  "segment/count" : ["dataSource"],
-  "segment/overShadowed/count" : [],
-
-  "segment/max" : [],
-  "segment/used" : ["dataSource"],
-  "segment/usedPercent" : ["dataSource"],
-
-  "jvm/pool/committed" : ["poolKind", "poolName"],
-  "jvm/pool/init" :["poolKind", "poolName"],
-  "jvm/pool/max" : ["poolKind", "poolName"],
-  "jvm/pool/used" : ["poolKind", "poolName"],
-  "jvm/bufferpool/count" : ["bufferPoolName"],
-  "jvm/bufferpool/used" : ["bufferPoolName"],
-  "jvm/bufferpool/capacity" : ["bufferPoolName"],
-  "jvm/mem/init" : ["memKind"],
-  "jvm/mem/max" : ["memKind"],
-  "jvm/mem/used" : ["memKind"],
-  "jvm/mem/committed" : ["memKind"],
-  "jvm/gc/count" : ["gcName"],
-  "jvm/gc/time" : ["gcName"],
-
-  "ingest/events/buffered" : ["dataSource"],
-  "ingest/kafka/lag" : ["dataSource"],
-
-  "sys/swap/free" : [],
-  "sys/swap/max" : [],
-  "sys/swap/pageIn" :[],
-  "sys/swap/pageOut" : [],
-  "sys/disk/write/count" : ["fsDevName"],
-  "sys/disk/read/count" : ["fsDevName"],
-  "sys/disk/write/size" : ["fsDevName"],
-  "sys/disk/read/size" : ["fsDevName"],
-  "sys/net/write/size" : [],
-  "sys/net/read/size" : [],
-  "sys/fs/used" : ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"],
-  "sys/fs/max" :  ["fsDevName", "fsDirName", "fsTypeName", "fsSysTypeName", "fsOptions"],
-  "sys/mem/used" : [],
-  "sys/mem/max" : [],
-  "sys/storage/used" :  ["fsDirName"],
-  "sys/cpu" :  ["cpuName", "cpuTime"],
-
-  "coordinator-segment/count" :  ["dataSource"],
-  "historical-segment/count" :  ["dataSource", "tier", "priority"]
-}
\ No newline at end of file
+  "segment/usedPercent" : ["dataSource"]
+}
diff --git a/ansible/roles/analytics-druid/meta/main.yaml b/ansible/roles/analytics-druid/meta/main.yaml
new file mode 100644
index 0000000000..2d92bdf2e3
--- /dev/null
+++ b/ansible/roles/analytics-druid/meta/main.yaml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: graphite-exporter
diff --git a/ansible/roles/analytics-druid/tasks/install.yml b/ansible/roles/analytics-druid/tasks/install.yml
index 3bba463166..cb912e8efa 100644
--- a/ansible/roles/analytics-druid/tasks/install.yml
+++ b/ansible/roles/analytics-druid/tasks/install.yml
@@ -1,5 +1,7 @@
 ---
 # file: install.yml
+- name: Set the druid configs
+  set_fact: druid_configs="{{ default_druid_configs | combine(private_druid_configs,recursive=True) if private_druid_configs is defined else default_druid_configs }}"
 
 - name: Create druid user
   user:
@@ -104,7 +106,7 @@
     owner: "{{ druid_user }}"
     mode: 755
 
-- name: Copy over systemd service file for druid role {{ druid_role }}
+- name: Copy over systemd service file for druid role {{cluster}}-{{ druid_role }}
   template: src=druid.service dest=/lib/systemd/system/druid_{{ druid_role }}.service
 
 - name: Start druid {{ druid_role }} service
diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
index 464c2d4e48..8a047b09ff 100644
--- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
+++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties
@@ -38,23 +38,52 @@ druid.zk.paths.base=/druid
 # Metadata storage
 # For PostgreSQL:
 druid.metadata.storage.type=postgresql
-druid.metadata.storage.connector.connectURI=jdbc:postgresql://{{ druid_postgres_host }}:{{ druid_postgres_port }}/{{ druid_postgres_db }}
-druid.metadata.storage.connector.user={{ druid_postgres_user }}
-druid.metadata.storage.connector.password={{ druid_postgres_pass }}
+druid.metadata.storage.connector.connectURI=jdbc:postgresql://{{ druid_configs[cluster].druid_postgres_host }}:{{ druid_configs[cluster].druid_postgres_port }}/{{ druid_configs[cluster].druid_postgres_db }}
+druid.metadata.storage.connector.user={{ druid_configs[cluster].druid_postgres_user }}
+druid.metadata.storage.connector.password={{ druid_configs[cluster].druid_postgres_pass }}
 
 # Deep storage
 #Using azure
-druid.storage.type={{ druid_storge_type }}
-druid.azure.account={{sunbird_druid_storage_account_name}}
-druid.azure.key={{sunbird_druid_storage_account_key}}
-druid.azure.container={{ azure_container }}
+druid.storage.type={{ druid_storage_type }}
+{% if druid_storage_type == "azure" %}
+druid.azure.account={{ druid_configs[cluster].azure_account_name }}
+druid.azure.key={{ druid_configs[cluster].azure_storage_secret }}
+druid.azure.container={{ druid_configs[cluster].azure_container }}
+{% elif druid_storage_type == "s3" %}
+druid.storage.bucket={{ druid_configs[cluster].s3_bucket }}
+druid.storage.baseKey={{ druid_configs[cluster].s3_segment_dir }}
+druid.s3.accessKey={{ druid_configs[cluster].s3_access_key }}
+druid.s3.secretKey={{ druid_configs[cluster].s3_secret_key }}
+# set protocol and endpoint together
+druid.s3.endpoint.url={{ druid_configs[cluster].s3_endpoint }}
+# to enable path like access
+# if true,  url=<protocol>://<host>/<bucket>
+# if false, url=<protocol>://<bucket>.<host>
+druid.s3.enablePathStyleAccess={{ druid_configs[cluster].s3_path_like_access }}
+
+# enable v4 signing of requests
+druid.s3.endpoint.signingRegion={{ druid_configs[cluster].s3_v4_sign_region }}
+
+# uncomment to enable access of bucket from any region
+# druid.s3.forceGlobalBucketAccessEnabled=true
+# uncomment to enable server side encryption for s3
+# druid.storage.sse.type=s3
+# uncomment to disable chunk encoding
+# druid.s3.disableChunkedEncoding=true
+{% endif %}
 
 # Indexing service logs
 # For local disk (only viable in a cluster if this is a network mount):
 druid.indexer.logs.type={{ druid_indexing_logs_type }}
+{% if druid_indexing_logs_type == "azure" %}
 #druid.indexer.logs.directory={{ druid_indexing_log_dir }}
-druid.indexer.logs.container={{ druid_log_azure_container }}
-druid.indexer.logs.prefix= {{ druid_log_azure_folder }}
+druid.indexer.logs.container={{ druid_configs[cluster].druid_log_azure_container }}
+druid.indexer.logs.prefix= {{ druid_configs[cluster].druid_log_azure_folder }}
+{% elif druid_indexing_logs_type == "s3" %}
+druid.indexer.logs.s3Bucket={{ druid_configs[cluster].s3_logging_bucket }}
+# path to logs within the bucker
+druid.indexer.logs.s3Prefix={{ druid_configs[cluster].s3_indexer_logs_dir }}
+{% endif %}
 
 # Service discovery
 druid.selectors.indexing.serviceName={{ druid_overlord_service }}
@@ -66,12 +95,14 @@ druid.emitter=composing
 druid.emitter.composing.emitters=[ {{ druid_common_emitters }}]
 druid.emitter.logging.logLevel=info
 
-{% if druid_graphite_host is defined %}
-druid.emitter.graphite.port={{ druid_graphite_port }}
-druid.emitter.graphite.hostname={{ druid_graphite_host }}
+{% if druid_monitoring %}
+druid.emitter.graphite.port={{ druid_graphite_port | d(9109) }}
+druid.emitter.graphite.hostname={{ druid_graphite_host | d("localhost") }}
+druid.emitter.graphite.protocol=plaintext
 druid.emitter.graphite.eventConverter={"type":"whiteList", "namespacePrefix": "{{ druid_graphite_prefix }}", "ignoreHostname":false, "ignoreServiceName":false ,"mapPath":"{{ druid_whitelist_filepath }}"}
 {% endif %}
 
+
 # Storage type of double columns
 # ommiting this will lead to index double as float at the storage layer
 
@@ -82,3 +113,4 @@ druid.request.logging.type={{ druid_request_logging_type }}
 druid.request.logging.dir={{ druid_log_dir }}
 
 druid.javascript.enabled=true
+druid.sql.enable={{ enable_druid_sql | lower }}
diff --git a/ansible/roles/analytics-druid/templates/broker/jvm.config b/ansible/roles/analytics-druid/templates/broker/jvm.config
index 530561493a..ccd6e12633 100644
--- a/ansible/roles/analytics-druid/templates/broker/jvm.config
+++ b/ansible/roles/analytics-druid/templates/broker/jvm.config
@@ -1,7 +1,7 @@
 -server
--Xms{{ druid_broker_heap_size }}
--Xmx{{ druid_broker_heap_size }}
--XX:MaxDirectMemorySize={{ druid_broker_max_direct_size }}
+-Xms{{ druid_configs[cluster].druid_broker_min_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_broker_min_heap_size }}
+-XX:MaxDirectMemorySize={{ druid_configs[cluster].druid_broker_max_direct_size }}
 -XX:+PrintGCDetails
 -XX:+UseG1GC
 -XX:+PrintGCTimeStamps
diff --git a/ansible/roles/analytics-druid/templates/broker/runtime.properties b/ansible/roles/analytics-druid/templates/broker/runtime.properties
index bb4ce59d99..b8d3d8842e 100644
--- a/ansible/roles/analytics-druid/templates/broker/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/broker/runtime.properties
@@ -3,12 +3,12 @@ druid.host={{ ansible_host }}
 druid.port={{ druid_broker_port }}
 
 # HTTP server threads
-druid.broker.http.numConnections={{ druid_broker_http_numConnections }}
-druid.server.http.numThreads={{ druid_broker_server_http_numThread }}
+druid.broker.http.numConnections={{ druid_configs[cluster].druid_broker_http_numConnections }}
+druid.server.http.numThreads={{ druid_configs[cluster].druid_broker_server_http_numThread }}
 
 # Processing threads and buffers
-druid.processing.buffer.sizeBytes={{ druid_broker_processing_bufferBytes }}
-druid.processing.numThreads={{ druid_broker_processing_threads }}
+druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_broker_processing_bufferBytes }}
+druid.processing.numThreads={{ druid_configs[cluster].druid_broker_processing_threads }}
 
 druid.javascript.enabled=true
 druid.sql.enable=true
diff --git a/ansible/roles/analytics-druid/templates/coordinator/jvm.config b/ansible/roles/analytics-druid/templates/coordinator/jvm.config
index dc6d765efb..ef3855dd52 100644
--- a/ansible/roles/analytics-druid/templates/coordinator/jvm.config
+++ b/ansible/roles/analytics-druid/templates/coordinator/jvm.config
@@ -1,6 +1,6 @@
 -server
--Xms{{ druid_coordinator_heap_size }}
--Xmx{{ druid_coordinator_heap_size }}
+-Xms{{ druid_configs[cluster].druid_coordinator_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_coordinator_heap_size }}
 -XX:+UseG1GC
 -XX:+PrintGCDetails 
 -XX:+PrintGCDateStamps
diff --git a/ansible/roles/analytics-druid/templates/coordinator/runtime.properties b/ansible/roles/analytics-druid/templates/coordinator/runtime.properties
index 8c176e83d9..a4a99948ed 100644
--- a/ansible/roles/analytics-druid/templates/coordinator/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/coordinator/runtime.properties
@@ -2,10 +2,11 @@ druid.service={{ druid_coordinator_service }}
 druid.host={{ ansible_host }}
 druid.port={{ druid_coordinator_port }}
 
-druid.coordinator.startDelay={{ druid_coordinator_startDelay }}
-druid.coordinator.period={{ druid_coordinator_period }}
+druid.coordinator.startDelay={{ druid_configs[cluster].druid_coordinator_startDelay }}
+druid.coordinator.period={{ druid_configs[cluster].druid_coordinator_period }}
+druid.coordinator.balancer.strategy={{ druid_configs[cluster].druid_coordinator_balance_strategy }}
 
 {% if druid_coordinator_monitoring_monitors is defined %}
-# Logging monitors
+#Logging monitors
 druid.monitoring.monitors=[{{ druid_coordinator_monitoring_monitors }}]
 {% endif %}
diff --git a/ansible/roles/analytics-druid/templates/druid.service b/ansible/roles/analytics-druid/templates/druid.service
index 8e7503b812..c36c4ed5d0 100644
--- a/ansible/roles/analytics-druid/templates/druid.service
+++ b/ansible/roles/analytics-druid/templates/druid.service
@@ -5,6 +5,8 @@ After=network.target
 [Service]
 WorkingDirectory={{ druid_path }}
 ExecStart=/bin/sh -c "/usr/bin/java $(/bin/cat {{ druid_path }}conf/druid/{{ service }}/jvm.config | /usr/bin/xargs) -cp {{ druid_path }}conf/druid/{{ service }}:{{ druid_path }}conf/druid/_common:{{ druid_path }}lib/* org.apache.druid.cli.Main server {{ service }}"
+LimitNOFILE={{ druid_open_file_limit }}
+LimitNPROC={{ druid_process_limit }}
 Restart=always
 
 [Install]
diff --git a/ansible/roles/analytics-druid/templates/historical/jvm.config b/ansible/roles/analytics-druid/templates/historical/jvm.config
index f076e6af60..18fd757a37 100644
--- a/ansible/roles/analytics-druid/templates/historical/jvm.config
+++ b/ansible/roles/analytics-druid/templates/historical/jvm.config
@@ -1,9 +1,10 @@
 -server
--Xms{{ druid_historical_heap_size }}
--Xmx{{ druid_historical_heap_size }}
--XX:MaxDirectMemorySize={{ druid_historical_max_direct_size }}
+-Xms{{ druid_configs[cluster].druid_historical_min_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_historical_max_heap_size }}
+-XX:MaxDirectMemorySize={{ druid_configs[cluster].druid_historical_max_direct_size }}
 -XX:+UseG1GC
 -XX:+HeapDumpOnOutOfMemoryError
+-XX:+ExitOnOutOfMemoryError
 -XX:+PrintGCDetails
 -XX:+PrintGCDateStamps
 -XX:+PrintGCTimeStamps
diff --git a/ansible/roles/analytics-druid/templates/historical/runtime.properties b/ansible/roles/analytics-druid/templates/historical/runtime.properties
index ee8e889d7c..3bad72e539 100644
--- a/ansible/roles/analytics-druid/templates/historical/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/historical/runtime.properties
@@ -4,27 +4,35 @@ druid.port={{ druid_historical_port }}
 
 
 # HTTP server threads
-druid.server.http.numThreads={{ druid_historical_server_http_numThread }}
+druid.server.http.numThreads={{ druid_configs[cluster].druid_historical_server_http_numThread }}
 
 # Processing threads and buffers
-druid.processing.buffer.sizeBytes={{ druid_historical_processing_bufferBytes }}
-druid.processing.numThreads={{ druid_historical_processing_threads }}
-druid.processing.numMergeBuffers={{ druid_historical_processing_num_merge_buffers }}
+druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_historical_processing_bufferBytes }}
+druid.processing.numThreads={{ druid_configs[cluster].druid_historical_processing_threads }}
+druid.processing.numMergeBuffers={{ druid_configs[cluster].druid_historical_processing_num_merge_buffers }}
 
-# Segment storage
-druid.segmentCache.locations=[{"path": "{{ druid_historical_segmentcache_path }}", "maxSize": {{ druid_historical_segmentcache_size }}}]
-druid.server.maxSize={{ druid_historical_server_maxsize }}
+# Segmentstorage
+#druid.segmentCache.locations=[{"path": "{{ druid_historical_segmentcache_path }}", "maxSize": {{ druid_configs[cluster].druid_historical_segmentcache_size }}}]
+druid.segmentCache.locations=[{% for number in range(5) %}{"path": "{{ druid_configs[cluster].druid_historical_segmentcache_path }}{{number }}", "maxSize": {{ druid_configs[cluster].druid_historical_segmentcache_size }}}{% if not loop.last %}, {% endif %}
+{% endfor %}]
 
+druid.segmentCache.numLoadingThreads={{ druid_configs[cluster].druid_historical_segmentcache_numloadingthreads }}
 
 {% if druid_historical_monitoring_monitors is defined %}
 # Monitors
 druid.monitoring.monitors=[{{ druid_historical_monitoring_monitors }}]
 {% endif %}
 
+{% if druid_configs[cluster].druid_query_ondiskstorage_enabled %}
+druid.query.groupBy.maxMergingDictionarySize={{ druid_configs[cluster].druid_historical_maxMergingDictionarySize }}
+druid.query.groupBy.maxOnDiskStorage={{ druid_configs[cluster].druid_historical_maxOnDiskStorage }}
+{% endif %}
+
+
 # Caching
-druid.historical.cache.useCache={{ druid_historical_enable_cache }} 
+druid.historical.cache.useCache={{ druid_configs[cluster].druid_historical_enable_cache }} 
 druid.historical.cache.populateCache=true
 druid.historical.cache.unCacheable=["select","scan"]
 druid.cache.type=caffeine
 druid.cache.sizeInBytes={{ druid_historical_cache_size }}
-druid.cache.expireAfter={{ druid_historical_cache_expiry }}
\ No newline at end of file
+druid.cache.expireAfter={{ druid_historical_cache_expiry }}
diff --git a/ansible/roles/analytics-druid/templates/middlemanager/jvm.config b/ansible/roles/analytics-druid/templates/middlemanager/jvm.config
index 7a9c216f2e..9d8c703052 100644
--- a/ansible/roles/analytics-druid/templates/middlemanager/jvm.config
+++ b/ansible/roles/analytics-druid/templates/middlemanager/jvm.config
@@ -1,6 +1,6 @@
 -server
--Xmx{{ druid_middlemanager_heap_size }}
--Xms{{ druid_middlemanager_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_middlemanager_heap_size }}
+-Xms{{ druid_configs[cluster].druid_middlemanager_heap_size }}
 -XX:+PrintGCDetails
 -XX:+UseG1GC
 -XX:+PrintGCTimeStamps
diff --git a/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties b/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties
index 0559cb03b4..6234b38f16 100644
--- a/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/middlemanager/runtime.properties
@@ -3,17 +3,16 @@ druid.host={{ ansible_host }}
 druid.port={{ druid_middlemanager_port }}
 
 # Number of tasks per middleManager
-druid.worker.capacity={{ druid_middlemanager_worker_cap }}
+druid.worker.capacity={{ druid_configs[cluster].druid_middlemanager_worker_cap }}
 
 # Task launch parameters
-druid.indexer.runner.javaOpts={{ druid_mm_java_opts_array }}
+druid.indexer.runner.javaOpts={{ druid_configs[cluster].druid_mm_java_opts_array }}
 druid.indexer.task.baseTaskDir={{ druid_indexing_task_basedir }}
 
 # Peon properties
-druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ druid_middlemanager_peon_processing_bufferBytes }}
-druid.indexer.fork.property.druid.processing.numThreads={{ druid_middlemanager_peon_processing_threads }}
-druid.indexer.fork.property.druid.segmentCache.locations=[{"path": "{{ druid_middlemanager_peon_segmentcache_path }}", "maxSize": {{ druid_middlemanager_peon_server_maxsize }}}]
-druid.indexer.fork.property.druid.server.http.numThreads={{ druid_middlemanager_peon_server_http_numThread }}
+druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ druid_configs[cluster].druid_middlemanager_peon_processing_bufferBytes }}
+druid.indexer.fork.property.druid.processing.numThreads={{ druid_configs[cluster].druid_middlemanager_peon_processing_threads }}
+druid.indexer.fork.property.druid.server.http.numThreads={{ druid_configs[cluster].druid_middlemanager_peon_server_http_numThread }}
 
 
 {% if druid_middlemanager_monitoring_monitors is defined %}
diff --git a/ansible/roles/analytics-druid/templates/overlord/jvm.config b/ansible/roles/analytics-druid/templates/overlord/jvm.config
index fb642350e5..75be38a1d4 100644
--- a/ansible/roles/analytics-druid/templates/overlord/jvm.config
+++ b/ansible/roles/analytics-druid/templates/overlord/jvm.config
@@ -1,6 +1,6 @@
 -server
--Xms{{ druid_overlord_heap_size }}
--Xmx{{ druid_overlord_heap_size }}
+-Xms{{ druid_configs[cluster].druid_overlord_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_overlord_heap_size }}
 -XX:+UseG1GC
 -XX:+PrintGCDetails
 -XX:+PrintGCTimeStamps
diff --git a/ansible/roles/analytics-druid/templates/overlord/runtime.properties b/ansible/roles/analytics-druid/templates/overlord/runtime.properties
index 314fc008cc..ac9d3eeec2 100644
--- a/ansible/roles/analytics-druid/templates/overlord/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/overlord/runtime.properties
@@ -2,7 +2,7 @@ druid.service={{ druid_overlord_service }}
 druid.port={{ druid_overlord_port }}
 druid.host={{ ansible_host }}
 
-druid.indexer.queue.startDelay={{ druid_indexing_queue_startDelay }}
+druid.indexer.queue.startDelay={{ druid_configs[cluster].druid_indexing_queue_startDelay }}
 
 druid.indexer.runner.type=remote
 druid.indexer.storage.type={{ druid_indexing_storage_type }}
diff --git a/ansible/roles/analytics-druid/templates/router/jvm.config b/ansible/roles/analytics-druid/templates/router/jvm.config
index a50bef1a10..dd55499200 100644
--- a/ansible/roles/analytics-druid/templates/router/jvm.config
+++ b/ansible/roles/analytics-druid/templates/router/jvm.config
@@ -1,10 +1,13 @@
 -server
--Xms{{ druid_router_heap_size }}
--Xmx{{ druid_router_heap_size }}
--XX:+UseG1GC
--XX:+ExitOnOutOfMemoryError
+-Xms{{ druid_configs[cluster].druid_router_heap_size }}
+-Xmx{{ druid_configs[cluster].druid_router_heap_size }}
 -XX:+UseG1GC
+-XX:+HeapDumpOnOutOfMemoryError
+-XX:+PrintGCTimeStamps
+-Xloggc:{{ druid_router_gc_logfile }}
+-XX:HeapDumpPath={{ druid_router_heap_dump_file }}
 -Duser.timezone=UTC
 -Dfile.encoding=UTF-8
--Djava.io.tmpdir=var/tmp
--Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
\ No newline at end of file
+-Djava.io.tmpdir={{ druid_router_tmp_dir }}
+-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+-Dlog4j.configurationFile={{ druid_path }}conf/druid/{{ service }}/log4j2.xml
\ No newline at end of file
diff --git a/ansible/roles/analytics-druid/templates/router/runtime.properties b/ansible/roles/analytics-druid/templates/router/runtime.properties
index 85f3395f8c..e7144c9416 100644
--- a/ansible/roles/analytics-druid/templates/router/runtime.properties
+++ b/ansible/roles/analytics-druid/templates/router/runtime.properties
@@ -17,18 +17,18 @@
 # under the License.
 #
 
-druid.service=druid/router
-druid.plaintextPort=8888
+druid.service={{ druid_router_service }}
+druid.plaintextPort={{ druid_router_port }}
 
 # HTTP proxy
-druid.router.http.numConnections={{ druid_router_http_numConnections }}
-druid.router.http.readTimeout={{ druid_router_http_readTimeout }}
-druid.router.http.numMaxThreads={{ druid_router_http_numMaxThreads }}
-druid.server.http.numThreads={{ druid_server_http_numThreads }}
+druid.router.http.numConnections={{ druid_configs[cluster].druid_router_http_numConnections }}
+druid.router.http.readTimeout={{ druid_configs[cluster].druid_router_http_readTimeout }}
+druid.router.http.numMaxThreads={{ druid_configs[cluster].druid_router_http_numMaxThreads }}
+druid.server.http.numThreads={{ druid_configs[cluster].druid_server_http_numThreads }}
 
 # Service discovery
-druid.router.defaultBrokerServiceName=druid/broker
-druid.router.coordinatorServiceName=druid/coordinator
+druid.router.defaultBrokerServiceName={{ druid_broker_service }}
+druid.router.coordinatorServiceName={{ druid_coordinator_service }}
 
 # Management proxy to coordinator / overlord: required for unified web console.
-druid.router.managementProxy.enabled={{ druid_router_managementProxy_enabled }}
+druid.router.managementProxy.enabled={{ druid_configs[cluster].druid_router_managementProxy_enabled }}
diff --git a/ansible/roles/analytics-druid/tests/test.yml b/ansible/roles/analytics-druid/tests/test.yml
index 85778d7824..bf02bb68f4 100644
--- a/ansible/roles/analytics-druid/tests/test.yml
+++ b/ansible/roles/analytics-druid/tests/test.yml
@@ -23,3 +23,4 @@
     - {role: analytics-druid,when: "'broker' in group_names",druid_role: 'broker', service: 'broker'}
     - {role: analytics-druid,when: "'historical' in group_names",druid_role: 'historical', service: 'historical'}
     - {role: analytics-druid,when: "'middlemanager' in group_names",druid_role: 'middlemanager',service: 'middleManager' }
+    - {role: analytics-druid,when: "'router' in group_names",druid_role: 'router',service: 'router' }
diff --git a/ansible/roles/analytics-run-report/defaults/main.yml b/ansible/roles/analytics-run-report/defaults/main.yml
index 5827623f2f..0a3913e7ff 100644
--- a/ansible/roles/analytics-run-report/defaults/main.yml
+++ b/ansible/roles/analytics-run-report/defaults/main.yml
@@ -4,7 +4,7 @@ analytics:
   home: /mount/data/analytics
 
 model_version: "2.0"
-spark_version: "2.4.4"
+spark_version: "3.1.3"
 
 pause_min: 0
 
diff --git a/ansible/roles/analytics-run-report/tasks/main.yml b/ansible/roles/analytics-run-report/tasks/main.yml
index 34a6bc9120..e939316e7c 100644
--- a/ansible/roles/analytics-run-report/tasks/main.yml
+++ b/ansible/roles/analytics-run-report/tasks/main.yml
@@ -1,5 +1,5 @@
 - name: Get the report_config
-  shell: 'curl -H "Content-Type: application/json" http://{{ analytics_host }}:9000/report/jobs/{{ report_id }}'
+  shell: 'curl -H "Content-Type: application/json" -H "Authorization: {{ druid_report_token }}" {{ druid_report_url_endpoint }}/{{ report_id }}'
   register : output
 
 - set_fact:
@@ -8,6 +8,22 @@
 - set_fact:
    report_config: "{{ config.result.config | to_json }}"
 
+- set_fact:
+    dynamic: true
+  when: report_start_date is defined  and report_end_date is defined  and report_start_date != ""  and report_end_date != ""
+
+- set_fact:
+    dateRange: "{{config.result.config.reportConfig.dateRange}}"
+  when: dynamic is defined and dynamic == true
+
+- name: Load the dynamic interval json
+  set_fact: {"dynamic_interval":"{{ lookup('template', 'interval.j2')}}"}
+  when: dynamic is defined and dynamic == true
+
+- set_fact:
+   report_config: "{{ config.result.config.reportConfig | combine(dynamic_interval) | to_json  }}"
+  when: dynamic is defined and dynamic == true
+
 - name: Update report config
   template: src=config.json.j2 dest={{ analytics.home }}/scripts/report_config.json mode=755 owner={{ analytics_user }} group={{ analytics_group }}
 
diff --git a/ansible/roles/analytics-run-report/templates/druid-report-job.j2 b/ansible/roles/analytics-run-report/templates/druid-report-job.j2
index 713a4f74b9..e9a68a6b3d 100644
--- a/ansible/roles/analytics-run-report/templates/druid-report-job.j2
+++ b/ansible/roles/analytics-run-report/templates/druid-report-job.j2
@@ -6,12 +6,13 @@ export SCRIPTS_HOME={{ analytics.home }}/scripts
 export DP_LOGS={{ analytics.home }}/logs/data-products
 cd {{ analytics.home }}/scripts
 
+today=$(date "+%Y-%m-%d")
 libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0"
 
 job_config=`cat $SCRIPTS_HOME/report_config.json`
 
 echo "Job Config - $job_config" >> "$DP_LOGS/$today-job-execution.log"
 
-nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "druid_reports" --config "$job_config"  >> "$DP_LOGS/$today-job-execution.log" 2>&1
+nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.12-2.5.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "druid_reports" --config "$job_config"  >> "$DP_LOGS/$today-job-execution.log" 2>&1
 
 echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log"
diff --git a/ansible/roles/analytics-run-report/templates/interval.j2 b/ansible/roles/analytics-run-report/templates/interval.j2
new file mode 100644
index 0000000000..0901b42ad8
--- /dev/null
+++ b/ansible/roles/analytics-run-report/templates/interval.j2
@@ -0,0 +1 @@
+{"dateRange":{"interval":{"startDate":"{{ report_start_date }}","endDate":"{{ report_end_date }}"},"granularity":"{{ dateRange.granularity | default('all') }}","intervalSlider":{{ dateRange.intervalSlider | default(0) | int }}}}
diff --git a/ansible/roles/analytics-spark-provision/defaults/main.yml b/ansible/roles/analytics-spark-provision/defaults/main.yml
index c05b0addaf..ef986228fe 100644
--- a/ansible/roles/analytics-spark-provision/defaults/main.yml
+++ b/ansible/roles/analytics-spark-provision/defaults/main.yml
@@ -2,8 +2,8 @@ analytics:
    home: /mount/data/analytics
    soft_path: /mount/data/analytics
    base_path: /home/analytics 
-scala_version: 2.11.8
-spark_version: 2.4.4
+scala_version: 2.12.10
+spark_version: 3.1.3
 model_version: "2.0" 
 spark:
   home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7"
@@ -32,3 +32,10 @@ hadoop_aws_url: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/2.7.
 java_xmlbuilder_url: https://repo1.maven.org/maven2/com/jamesmurty/utils/java-xmlbuilder/1.1/java-xmlbuilder-1.1.jar
 hadoop_azure_url: https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-azure/2.7.3/hadoop-azure-2.7.3.jar
 azure_storage: https://repo1.maven.org/maven2/com/microsoft/azure/azure-storage/3.0.0/azure-storage-3.0.0.jar
+
+# jets3t s3 config, allows us to configure for s3-like object stores
+jets3t_s3_request_signature_version: "{{ s3_request_signature_version }}"
+jets3t_s3_endpoint_host: "{% if s3_storage_endpoint %}{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}{% endif %}"
+jets3t_s3_disable_dns_buckets: "{{ s3_path_style_access }}"
+jets3t_s3_https_only: "{{ s3_https_only }}"
+jets3t_s3_default_bucket_location: "{{ s3_default_bucket_location }}"
diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml
index 07bd81d8a8..25ebd9da23 100644
--- a/ansible/roles/analytics-spark-provision/tasks/main.yml
+++ b/ansible/roles/analytics-spark-provision/tasks/main.yml
@@ -10,12 +10,12 @@
   become_user: "{{ analytics_user }}"
   unarchive: src={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7
 
-- name: Download Scala 2.11.8
+- name: Download Scala 2.12.10
   become: yes
   become_user: "{{ analytics_user }}"
   shell: wget -O {{ analytics.soft_path }}/scala-{{ scala_version }}.tgz https://downloads.lightbend.com/scala/{{ scala_version }}/scala-{{ scala_version }}.tgz
 
-- name: Unarchive Scala 2.11.8
+- name: Unarchive Scala 2.12.10
   become: yes
   become_user: "{{ analytics_user }}"
   unarchive: src={{ analytics.soft_path }}/scala-{{ scala_version }}.tgz dest={{ analytics.soft_path }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.soft_path }}/scala-{{ scala_version }}
@@ -82,51 +82,63 @@
   become_user: "{{ analytics_user }}"
   template: src=jets3t.j2 dest={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7/conf/jets3t.properties mode=755 owner={{ analytics_user }} group={{ analytics_group }}
 
-
 ## Provision DS ##
 - name: Install packages
   become: yes
   apt:
-    name: "{{ item }}"
+    name: ["gcc", "autoconf", "curl", "g++", "gnupg", "automake", "bison", "libc6-dev", "libffi-dev", "libgdbm-dev", "libncurses5-dev", "libsqlite3-dev", "pkg-config", "sqlite3", "zlib1g-dev", "libtool", "libyaml-dev", "make", "libgmp-dev", "libreadline-dev", "libssl-dev", "gpgv2", "gem", "ruby", "gnupg2", "virtualenv"]
     state: installed
     update_cache: true
     cache_valid_time: 3600
-  with_items:
-    - gem
-    - ruby
-    - gnupg2
 
 - name: Import rvm keys
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: command curl -sSL https://rvm.io/mpapis.asc | gpg2 --import - && curl -sSL https://rvm.io/pkuczynski.asc | gpg2 --import -
+  shell: gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
 
 - name: Install rvm
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: curl -sSL https://get.rvm.io | bash -s stable  
+  shell: curl -sSL https://get.rvm.io | bash -s stable
 
 - name: source ruby script
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: echo "source {{ analytics.base_path }}/.rvm/scripts/rvm" >> ~/.bashrc
-  args:
-    executable: /bin/bash
+  lineinfile:
+    path: "{{ analytics.base_path }}/.bashrc"
+    line: source /home/analytics/.rvm/scripts/rvm
+    create: yes
 
-- name: Install latest ruby
+- name: Change ownership
+  file:
+    path: "{{ analytics.base_path }}"
+    owner: "{{ analytics_user }}"
+    group: "{{ analytics_user }}"
+    recurse: yes
   become: yes
-  shell: "{{ analytics.base_path }}/.rvm/bin/rvm install ruby-2.6"
 
-- name: Set latest ruby as default
+- name: Install latest ruby
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: "{{ analytics.base_path }}/.rvm/bin/rvm --default use ruby-2.6"
+  shell: "export PATH=$PATH:/home/analytics/.rvm/bin && rvm install ruby-2.5"
+
+- name: Add ruby repository
+  become: yes
+  apt_repository:
+    repo: ppa:brightbox/ruby-ng
+
+- name: Install latest ruby-dev
+  become: yes
+  apt:
+    name: "ruby2.5-dev"
+    state: installed
+    update_cache: true
+    cache_valid_time: 3600
 
 - name: Install ruby-kafka
-  gem:
-     name: ruby-kafka
   become: yes
   become_user: "{{ analytics_user }}"
+  shell: "bash -ilc 'export PATH=$PATH:/home/analytics/.rvm/bin && rvm --default use ruby-2.5 && gem install ruby-kafka'"
 
 - name: Download Kafka-2.11
   become: yes
@@ -142,6 +154,15 @@
   tags:
     - kafka-provision
 
+- name: Add python ppa
+  become: yes
+  apt_repository:
+    repo: ppa:deadsnakes/ppa
 
-
-
+- name: Install python 3.6
+  become: yes
+  apt:
+    name: python3.6
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
diff --git a/ansible/roles/analytics-spark-provision/templates/jets3t.j2 b/ansible/roles/analytics-spark-provision/templates/jets3t.j2
index df9a223127..4de8480bc2 100644
--- a/ansible/roles/analytics-spark-provision/templates/jets3t.j2
+++ b/ansible/roles/analytics-spark-provision/templates/jets3t.j2
@@ -1,3 +1,8 @@
-storage-service.request-signature-version=AWS4-HMAC-SHA256
-s3service.s3-endpoint=s3-ap-south-1.amazonaws.com
+storage-service.request-signature-version={{ jets3t_s3_request_signature_version }}
+s3service.s3-endpoint={% if jets3t_s3_endpoint_host %}{{ jets3t_s3_endpoint_host }}{% else %}s3-ap-south-1.amazonaws.com{% endif %}
+s3service.disable-dns-buckets={{ jets3t_s3_disable_dns_buckets }}
+s3service.https-only={{ jets3t_s3_https_only }}
+{% if jets3t_s3_default_bucket_location %}
+s3service.default-bucket-location={{ jets3t_s3_default_bucket_location }}
+{% endif %}
 uploads.stream-retry-buffer-size=2147483646
diff --git a/ansible/roles/analytics-spark-provision/templates/spark-env.j2 b/ansible/roles/analytics-spark-provision/templates/spark-env.j2
index ad8c5c4e9b..dea6e5ad06 100644
--- a/ansible/roles/analytics-spark-provision/templates/spark-env.j2
+++ b/ansible/roles/analytics-spark-provision/templates/spark-env.j2
@@ -74,3 +74,8 @@ export reports_storage_key={{sunbird_private_storage_account_name}}
 export reports_storage_secret={{sunbird_private_storage_account_key}}
 export azure_storage_key={{sunbird_private_storage_account_name}}
 export azure_storage_secret={{sunbird_private_storage_account_key}}
+export druid_storage_account_key={{sunbird_public_storage_account_name}}
+export druid_storage_account_secret={{sunbird_public_storage_account_key}}
+export aws_storage_key={{ s3_storage_key }}
+export aws_storage_secret={{ s3_storage_secret }}
+
diff --git a/ansible/roles/ansible-kafka-upgrade/defaults/main.yml b/ansible/roles/ansible-kafka-upgrade/defaults/main.yml
index ec93841925..0fa8d0b5cf 100644
--- a/ansible/roles/ansible-kafka-upgrade/defaults/main.yml
+++ b/ansible/roles/ansible-kafka-upgrade/defaults/main.yml
@@ -2,7 +2,7 @@
 apache_mirror: http://apache.mirrors.tds.net
 apache_mirror_archive: https://archive.apache.org/dist
 kafka_hosts: "{{ groups['processing-cluster-kafka']|join(':9092,')}}:9092"
-kafka_version: 1.1.0
+kafka_version: 2.8.0
 kafka_version_old: 0.10.0.1
 kafka_scala_version: 2.12
 run_mode: Deploy
@@ -53,4 +53,4 @@ user: ecosystem
 kafka_systemd_service: /etc/systemd/system/kafka.service
 kafka_auto_create_topics: "false"
 delete_topic_enable: "true"
-kafka_openfile_limit: 32768
+openfile_limit: 40768
diff --git a/ansible/roles/ansible-kafka-upgrade/tasks/start.yml b/ansible/roles/ansible-kafka-upgrade/tasks/start.yml
index 814f45e4c5..26d9a1bb5f 100644
--- a/ansible/roles/ansible-kafka-upgrade/tasks/start.yml
+++ b/ansible/roles/ansible-kafka-upgrade/tasks/start.yml
@@ -3,6 +3,10 @@
   wait_for: host={{zookeeper_listen_address| default('localhost')}} port={{zookeeper_port}} state=started timeout=30
   when: verify
 
+- name: Waiting 1m for kafka to settle
+  pause:
+    minutes: 1
+
 - name: Enable and Start Kafka
   service: name=kafka state=started enabled=yes
   become: yes
diff --git a/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2 b/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2
index 29a97b924c..57ead422a9 100644
--- a/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2
+++ b/ansible/roles/ansible-kafka-upgrade/templates/server.properties.j2
@@ -123,13 +123,13 @@ log.cleaner.enable=false
 # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
 # You can also append an optional chroot string to the urls to specify the
 # root directory for all kafka znodes.
-zookeeper.connect={% for host in groups['zookeeper'] %}{{ hostvars[host].ansible_default_ipv4.address }}:{{zookeeper_port}}{% if not loop.last %},{% endif %}{% endfor %}
+zookeeper.connect={% for host in groups['zookeeper'] %}{{ hostvars[host].inventory_hostname }}:{{zookeeper_port}}{% if not loop.last %},{% endif %}{% endfor %}
 
 # Timeout in ms for connecting to zookeeper
 zookeeper.connection.timeout.ms={{zookeeper_connection_timeout_ms}}
 
-inter.broker.protocol.version={{kafka_version_old}}
-log.message.format.version={{kafka_version_old}}
+#inter.broker.protocol.version={{kafka_version}}
+#log.message.format.version={{kafka_version}}
 
 ############################# Group Coordinator Settings #############################
 
diff --git a/ansible/roles/aws-cloud-storage/defaults/main.yml b/ansible/roles/aws-cloud-storage/defaults/main.yml
new file mode 100644
index 0000000000..6f3f6f86d6
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/defaults/main.yml
@@ -0,0 +1,3 @@
+s3_bucket_name: ""
+s3_path: ""
+local_file_or_folder_path: ""
diff --git a/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml b/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml
new file mode 100644
index 0000000000..c912b14edb
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/delete-folder.yml
@@ -0,0 +1,9 @@
+---
+- name: delete files and folders recursively
+  environment:
+    AWS_DEFAULT_REGION: "{{ aws_default_region }}"
+    AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}"
+    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}"
+  shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive"
+  async: 3600
+  poll: 10
diff --git a/ansible/roles/aws-cloud-storage/tasks/delete.yml b/ansible/roles/aws-cloud-storage/tasks/delete.yml
new file mode 100644
index 0000000000..414ea52e6b
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/delete.yml
@@ -0,0 +1,9 @@
+---
+- name: delete files from s3
+  environment:
+    AWS_DEFAULT_REGION: "{{ aws_default_region }}"
+    AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}"
+    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}"
+  shell: "aws s3 rm s3://{{ s3_bucket_name }}/{{ s3_path }}"
+  async: 3600
+  poll: 10
diff --git a/ansible/roles/aws-cloud-storage/tasks/download.yml b/ansible/roles/aws-cloud-storage/tasks/download.yml
new file mode 100644
index 0000000000..138024af78
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/download.yml
@@ -0,0 +1,9 @@
+---
+- name: download files to s3
+  environment:
+    AWS_DEFAULT_REGION: "{{ aws_default_region }}"
+    AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}"
+    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}"
+  shell: "aws s3 cp s3://{{ s3_bucket_name }}/{{ s3_path }} {{ local_file_or_folder_path }}"
+  async: 3600
+  poll: 10
diff --git a/ansible/roles/aws-cloud-storage/tasks/main.yml b/ansible/roles/aws-cloud-storage/tasks/main.yml
new file mode 100644
index 0000000000..62f204a9d2
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/main.yml
@@ -0,0 +1,18 @@
+---
+- name: delete files from aws S3 bucket 
+  include: delete.yml
+  
+- name: delete folders from aws S3 bucket recursively
+  include: delete-folder.yml
+  
+
+- name: download file from S3
+  include: download.yml
+
+- name: upload files from a local to aws S3
+  include: upload.yml
+
+- name: upload files and  folder from local directory to aws S3
+  include: upload-folder.yml
+ 
+
diff --git a/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml b/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml
new file mode 100644
index 0000000000..3e03b068b7
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/upload-folder.yml
@@ -0,0 +1,9 @@
+---
+- name: upload folder to s3
+  environment:
+    AWS_DEFAULT_REGION: "{{ aws_default_region }}"
+    AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}"
+    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}"
+  shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }} --recursive"
+  async: 3600
+  poll: 10
diff --git a/ansible/roles/aws-cloud-storage/tasks/upload.yml b/ansible/roles/aws-cloud-storage/tasks/upload.yml
new file mode 100644
index 0000000000..af8de990e2
--- /dev/null
+++ b/ansible/roles/aws-cloud-storage/tasks/upload.yml
@@ -0,0 +1,9 @@
+---
+- name: upload files to s3
+  environment:
+    AWS_DEFAULT_REGION: "{{ aws_default_region }}"
+    AWS_ACCESS_KEY_ID: "{{ aws_access_key_id }}"
+    AWS_SECRET_ACCESS_KEY: "{{ aws_secret_access_key }}"
+  shell: "aws s3 cp {{ local_file_or_folder_path }} s3://{{ s3_bucket_name }}/{{ s3_path }}"
+  async: 3600
+  poll: 10
diff --git a/ansible/roles/azure-cli/tasks/main.yml b/ansible/roles/azure-cli/tasks/main.yml
index 6c05b3e9f6..60d22aae73 100644
--- a/ansible/roles/azure-cli/tasks/main.yml
+++ b/ansible/roles/azure-cli/tasks/main.yml
@@ -1,13 +1,19 @@
-#- name: Install azure cli
-#  shell: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
-#  become: yes
-- name: Add Azure apt repository
-  apt_repository: repo='deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ xenial main' state=present
-
 - name: Import Azure signing key
   become: yes
   shell: curl -L https://packages.microsoft.com/keys/microsoft.asc | apt-key add -
 
+- name: Add Azure apt repository
+  apt_repository: repo='deb [arch=amd64] https://packages.microsoft.com/repos/azure-cli/ {{ ansible_distribution_release }} main' state=present
+
+- name: Add distribution release security apt repository
+  apt_repository: repo='deb http://security.ubuntu.com/ubuntu bionic-security main' state=present
+
+- name: install azure cli dependency
+  apt: name={{ item }} state=present update_cache=yes 
+  with_items:
+    - libssl1.0-dev
+  when: ansible_distribution_release == "focal"
+
 - name: ensure azure-cli and apt-transport-https is installed
   apt: name={{ item }} state=present update_cache=yes 
   with_items:
diff --git a/ansible/roles/azure-cloud-storage/defaults/main.yml b/ansible/roles/azure-cloud-storage/defaults/main.yml
new file mode 100644
index 0000000000..0e4e45bf95
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/defaults/main.yml
@@ -0,0 +1,67 @@
+# The name of the blob container in the azure storage account
+# Example -
+# blob_container_name: "my-container"
+blob_container_name: ""
+
+# The delete pattern to delete files and folder
+# Example -
+# blob_delete_pattern: "my-drectory/*"
+# blob_delete_pattern: "my-drectory/another-directory/*"
+# blob_delete_pattern: "*"
+blob_delete_pattern: ""
+
+# The storage account name
+# Example -
+# storage_account_name: "sunbird-dev-public"
+storage_account_name: ""
+
+# The storage account key
+# Example -
+# storage_account_name: "cmFuZG9tcmFuZG9tcmFuZG9tcmFuZG9tCg=="
+storage_account_key: ""
+
+# The path to local file which has to be uploaded to azure storage
+# The local path to store the file after downloading from azure storage
+# Example -
+# local_file_or_folder_path: "/workspace/my-folder/myfile.json"
+# local_file_or_folder_path: "/workspace/my-folder"
+local_file_or_folder_path: ""
+
+# The name of the file in azure storage after uploading from local
+# The name of the file in azure storage that has to be downloaded
+# Example -
+# blob_file_name: "myfile-blob.json"
+# You can also pass folder path in order to upload / download the file from  a speciic folder
+# blob_file_name "my-folder/my-file.json"
+blob_file_name: ""
+
+# The storage account sas token
+# Example -
+# storage_account_sas_token: "?sv=2022-01-01&ss=abc&srt=rws%3D"
+storage_account_sas_token: ""
+
+# The folder path in azure storage to upload the files starting from the root of the container
+# This path should alwasy start with a slash / as we are going to append this value as shown in below example
+# Example -
+# blob_container_name: "my-container"
+# blob_container_folder_path: "/my-folder-path"
+# {{ blob_container_name }}{{ blob_container_folder_path }}
+# The above translates to "my-container/my-folder-path"
+
+# The variable can also be empty as shown below, which means we will upload directly at the root path of the container
+# Example -
+# blob_container_name: "my-container"
+# blob_container_folder_path: ""
+# The above translates to "my-container"
+blob_container_folder_path: ""
+
+# At what access level the container should be created
+# Example -
+# container_public_access: "off"
+# container_public_access: "blob"
+# container_public_access: "container"
+# Allowed values are - off, blob, container
+# This variable affects only new containers and has no affect on a container if it already exists
+# If the container already exists, the access level will not be changed
+# You will need to change the access level from Azure portal or using az storage container set-permission command
+container_public_access: ""
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/blob-delete-batch.yml b/ansible/roles/azure-cloud-storage/tasks/blob-delete-batch.yml
new file mode 100644
index 0000000000..4e8ad68a2d
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/blob-delete-batch.yml
@@ -0,0 +1,5 @@
+---
+- name: delete files and folders from a blob container recursively
+  shell: "az storage blob delete-batch --source {{ blob_container_name  }} --pattern '{{ blob_delete_pattern }}' --account-name {{ storage_account_name }} --account-key {{ storage_account_key }}"
+  async: 3600
+  poll: 10
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/blob-download.yml b/ansible/roles/azure-cloud-storage/tasks/blob-download.yml
new file mode 100644
index 0000000000..3bbf4b607a
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/blob-download.yml
@@ -0,0 +1,5 @@
+---
+- name: download a file from azure storage
+  shell: "az storage blob download --container-name {{ blob_container_name }} --file {{ local_file_or_folder_path }} --name {{ blob_file_name }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }}"
+  async: 3600
+  poll: 10
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/blob-upload-batch.yml b/ansible/roles/azure-cloud-storage/tasks/blob-upload-batch.yml
new file mode 100644
index 0000000000..3043da46cc
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/blob-upload-batch.yml
@@ -0,0 +1,10 @@
+---
+- name: create container in azure storage if it doesn't exist
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: container-create.yml
+
+- name: upload files and folders from a local directory to azure storage container
+  shell: "az storage blob upload-batch --destination {{ blob_container_name  }}{{ blob_container_folder_path }} --source {{ local_file_or_folder_path }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }}"
+  async: 3600
+  poll: 10
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/blob-upload.yml b/ansible/roles/azure-cloud-storage/tasks/blob-upload.yml
new file mode 100644
index 0000000000..4b493ffb73
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/blob-upload.yml
@@ -0,0 +1,10 @@
+---
+- name: create container in azure storage if it doesn't exist
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: container-create.yml
+
+- name: upload file to azure storage container
+  shell: "az storage blob upload --container-name {{ blob_container_name  }} --file {{ local_file_or_folder_path }} --name {{ blob_file_name }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }}"
+  async: 3600
+  poll: 10
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/container-create.yml b/ansible/roles/azure-cloud-storage/tasks/container-create.yml
new file mode 100644
index 0000000000..419510cc19
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/container-create.yml
@@ -0,0 +1,8 @@
+---
+- name: create container in azure storage if it doesn't exist
+  shell: "az storage container create --name {{ blob_container_name }} --public-access {{ container_public_access }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }}"
+  when: storage_account_key | length > 0
+
+- name: create container in azure storage if it doesn't exist
+  shell: "az storage container create --name {{ blob_container_name }} --public-access {{ container_public_access }} --account-name {{ storage_account_name }} --sas-token '{{ storage_account_sas_token }}'"
+  when: storage_account_sas_token | length > 0
\ No newline at end of file
diff --git a/ansible/roles/azure-cloud-storage/tasks/delete-using-azcopy.yml b/ansible/roles/azure-cloud-storage/tasks/delete-using-azcopy.yml
new file mode 100644
index 0000000000..196de9c9b3
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/delete-using-azcopy.yml
@@ -0,0 +1,17 @@
+---
+- name: generate SAS token for azcopy
+  shell: |
+   sas_expiry=`date -u -d "1 hour" '+%Y-%m-%dT%H:%MZ'`
+   sas_token=?`az storage container generate-sas -n {{ blob_container_name }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }} --https-only --permissions dlrw --expiry $sas_expiry -o tsv`
+   echo $sas_token
+  register: sas_token
+
+- set_fact:
+   container_sas_token: "{{ sas_token.stdout}}"
+
+- name: delete files and folders from azure storage using azcopy
+  shell: "azcopy rm 'https://{{ storage_account_name }}.blob.core.windows.net/{{ blob_container_name }}{{ blob_container_folder_path }}{{ container_sas_token }}' --recursive"
+  environment:
+    AZCOPY_CONCURRENT_FILES: "10"
+  async: 10800
+  poll: 10
diff --git a/ansible/roles/azure-cloud-storage/tasks/main.yml b/ansible/roles/azure-cloud-storage/tasks/main.yml
new file mode 100644
index 0000000000..eb435ecfe2
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/main.yml
@@ -0,0 +1,21 @@
+---
+- name: delete files and folders from azure storage container recursively
+  include: blob-delete-batch.yml
+
+- name: download a file from azure storage
+  include: blob-download.yml
+
+- name: upload files and folders from a local directory to azure storage container
+  include: blob-upload-batch.yml
+
+- name: upload file to azure storage container
+  include: blob-upload.yml
+
+- name: create container in azure storage if it doesn't exist
+  include: container-create.yml
+
+- name: delete files and folders from azure storage using azcopy
+  include: delete-using-azcopy.yml
+
+- name: upload files and folders to azure storage using azcopy
+  include: upload-using-azcopy.yml
diff --git a/ansible/roles/azure-cloud-storage/tasks/upload-using-azcopy.yml b/ansible/roles/azure-cloud-storage/tasks/upload-using-azcopy.yml
new file mode 100644
index 0000000000..d86b233ead
--- /dev/null
+++ b/ansible/roles/azure-cloud-storage/tasks/upload-using-azcopy.yml
@@ -0,0 +1,22 @@
+---
+- name: generate SAS token for azcopy
+  shell: |
+   sas_expiry=`date -u -d "1 hour" '+%Y-%m-%dT%H:%MZ'`
+   sas_token=?`az storage container generate-sas -n {{ blob_container_name }} --account-name {{ storage_account_name }} --account-key {{ storage_account_key }} --https-only --permissions dlrw --expiry $sas_expiry -o tsv`
+   echo $sas_token
+  register: sas_token
+
+- set_fact:
+   container_sas_token: "{{ sas_token.stdout}}"
+
+- name: create container in azure storage if it doesn't exist
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: container-create.yml
+
+- name: upload files and folders to azure storage using azcopy
+  shell: "azcopy copy {{ local_file_or_folder_path }} 'https://{{ storage_account_name }}.blob.core.windows.net/{{ blob_container_name }}{{ blob_container_folder_path }}{{ container_sas_token }}' --recursive"
+  environment:
+    AZCOPY_CONCURRENT_FILES: "10"
+  async: 10800
+  poll: 10
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml b/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml
new file mode 100644
index 0000000000..95b0b73e0d
--- /dev/null
+++ b/ansible/roles/azure-hdinsight-spark-cluster/defaults/main.yml
@@ -0,0 +1,7 @@
+ 
+spark-folder: /usr/hdp/current/spark2-client
+guava_version: 19.0
+log4j_version: 2.5
+guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar
+log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar
+log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml b/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml
new file mode 100644
index 0000000000..d8f4d3cc50
--- /dev/null
+++ b/ansible/roles/azure-hdinsight-spark-cluster/tasks/main.yml
@@ -0,0 +1,13 @@
+- name: copy cluster creation script
+  template:
+    src: create-cluster.sh.j2
+    dest: /tmp/create-cluster.sh
+    mode: 0755
+  when: cluster_state == "create_cluster"
+
+- name: copy cluster deletion script
+  template:
+    src: delete-cluster.sh.j2
+    dest: /tmp/delete-cluster.sh
+    mode: 0755
+  when: cluster_state == "delete_cluster"
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
new file mode 100644
index 0000000000..8a7703c936
--- /dev/null
+++ b/ansible/roles/azure-hdinsight-spark-cluster/templates/create-cluster.sh.j2
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+cluster_name="{{env}}-spark-cluster"
+resource_group="{{azure_resource_group}}"
+cluster_type=spark
+component_version="{{component_version}}"
+headnode_size="{{headnode_size}}"
+location="{{location}}"
+http_user=admin
+http_password="{{azure_spark_cluster_http_password}}"
+storage_account_name="{{sunbird_private_storage_account_name}}"
+storage_account_key="{{sunbird_private_storage_account_key}}"
+storage_container="{{spark_storage_container}}"
+subnet_name="{{subnet_name}}"
+vnet_name="{{vnet_name}}"
+version="{{version}}"
+headnode_size="{{headnode_size}}"
+workernode_size="{{workernode_size}}"
+workernode_count="{{workernode_count}}"
+ssh_user="{{ ansible_ssh_user }}"
+ssh_pub_key="{{ssh_public_key_deployer}}"
+#ssh_pass="{{azure_spark_cluster_http_password}}"
+
+
+
+#az login 
+#az account set --subscription {{subscription_id}}
+
+
+az login --service-principal -u $1 -p $2 --tenant {{tenant_id}}
+az account set --subscription {{subscription_id}}
+
+az hdinsight create --name $cluster_name --resource-group $resource_group --type $cluster_type --cluster-tier Standard --component-version Spark=$component_version --headnode-size $headnode_size --location $location --http-password $http_password --http-user $http_user --ssh-user $ssh_user --ssh-public-key "$ssh_pub_key" --storage-account $storage_account_name --storage-account-key $storage_account_key --storage-container $storage_container --subnet $subnet_name --vnet-name $vnet_name --version $version --workernode-count $workernode_count --workernode-size $workernode_size --tags Name=$cluster_name
+
+mapfile -t nic_array < <( az network nic list -g $resource_group -o table | grep 'headnode\|workernode' | awk '{print $5}' )
+chmod 0777 "{{inventory_dir}}/hosts"
+host_line=$(sed -n '/\[spark-hdinsight\]/=' "{{inventory_dir}}/hosts")
+del_line=$((host_line+1))
+sed -i $del_line'd' "{{inventory_dir}}/hosts"
+
+
+for nic in "${nic_array[@]}";
+do
+  tag_name=$(az network nic show -n  $nic -g $resource_group | grep "$cluster_name" | cut -d':' -f2 | tr -d '"' | tr -d ' ')
+  if [ "$tag_name" == "$cluster_name" ]; then
+      private_ip=$(az network nic show -n $nic -g $resource_group | grep '"privateIpAddress":' | cut -d':' -f 2 | cut -d ',' -f1  | tr -d '"' | tr -d ' ')
+      name=$(echo $nic | cut -d'-' -f2)
+      count=$(echo $nic | cut -d'-' -f3)
+      hostname=$name-$count
+      echo $hostname : $private_ip
+      sed -i '/^\[spark-hdinsight\]/a '$private_ip'' "{{inventory_dir}}/hosts"
+  fi
+done
diff --git a/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2 b/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2
new file mode 100644
index 0000000000..7407049a9a
--- /dev/null
+++ b/ansible/roles/azure-hdinsight-spark-cluster/templates/delete-cluster.sh.j2
@@ -0,0 +1,13 @@
+ 
+#!/bin/bash
+
+cluster_name="{{env}}-spark-cluster"
+resource_group="{{azure_resource_group}}"
+
+#az login
+#az account set --subscription {{subscription_id}}
+
+az login --service-principal -u $1 -p $2 --tenant {{tenant_id}}
+az account set --subscription {{subscription_id}}
+
+az hdinsight delete --name $cluster_name --resource-group $resource_group --yes
diff --git a/ansible/roles/cassandra-3.7.0/defaults/main.yml b/ansible/roles/cassandra-3.7.0/defaults/main.yml
deleted file mode 100644
index 32a1c74999..0000000000
--- a/ansible/roles/cassandra-3.7.0/defaults/main.yml
+++ /dev/null
@@ -1,16 +0,0 @@
-cassandra_repo: "deb http://debian.datastax.com/datastax-ddc 3.7 main"
-data_file_directories: /data/cassandra/data
-commitlog_directory: /data/cassandra/commitlog
-saved_caches_directory: /data/cassandra/saved_caches
-seeds: "{{ inventory_hostname }}"
-listen_address: "{{ inventory_hostname }}"
-broadcast_rpc_address: "{{ inventory_hostname }}"
-paths: ['/data/cassandra', '/data/cassandra/commitlog', '/data/cassandra/data', '/data/cassandra/saved_caches', '/data/cassandra/logs']
-user: cassandra
-cassandra_lib: "/usr/share/cassandra/lib/"
-cassandra_lib_src: "templates/cassandra-lib.tgz"
-cassandra_conf: "/etc/cassandra/"
-cassandra_env_file: "/etc/cassandra/cassandra-env.sh"
-user_home: /home/analytics
-keyspace_prefix: '{{ env }}_'
-replication_factor: 1
diff --git a/ansible/roles/cassandra-3.7.0/handlers/main.yml b/ansible/roles/cassandra-3.7.0/handlers/main.yml
deleted file mode 100644
index 60bf5669e9..0000000000
--- a/ansible/roles/cassandra-3.7.0/handlers/main.yml
+++ /dev/null
@@ -1,5 +0,0 @@
-# handlers file for ansible-cassandra
-- name: "restart cassandra"
-  service:
-    name: "cassandra"
-    state: "restarted"
\ No newline at end of file
diff --git a/ansible/roles/cassandra-3.7.0/tasks/main.yml b/ansible/roles/cassandra-3.7.0/tasks/main.yml
deleted file mode 100644
index 34c8c0ee57..0000000000
--- a/ansible/roles/cassandra-3.7.0/tasks/main.yml
+++ /dev/null
@@ -1,84 +0,0 @@
-## Cassandra Provisioning ##
-- name: Cassandra | Make sure the ansible required dependencies are installed
-  become: yes
-  apt: pkg={{ item }} state=present
-  with_items:
-    - python-pycurl
-    - python-pip
-
-- name: install cassandra driver
-  become: yes
-  pip: name=cassandra-driver state=present
-
-- name: export cqlsh variable
-  become: yes
-  lineinfile:
-    dest: /etc/environment
-    line: 'CQLSH_NO_BUNDLED=TRUE'
-    state: present
-
-- name: Cassandra | Add the datastax repository apt-key
-  become: yes
-  apt_key:
-    url: "http://debian.datastax.com/debian/repo_key"
-    state: present
-
-- name: Cassandra | Add the cassandra 3.7 repository
-  become: yes
-  apt_repository:
-    repo: "{{ cassandra_repo }}"
-    state: present
-
-- name: Cassandra | Install the cassandra package
-  become: yes
-  apt:
-    name: cassandra
-    state: present
-    update_cache: yes
-    allow_unauthenticated: yes
-
-- name: Cassandra | Update the cassandra configuration
-  become: yes
-  template:
-    src: "cassandra.j2"
-    dest: "/etc/cassandra/cassandra.yaml"
-    mode: 0644
-
-- name: Cassandra | Update the cassandra rackdc configuration
-  become: yes
-  template:
-    src: "cassandra-rackdc.properties.j2"
-    dest: "/etc/cassandra/cassandra-rackdc.properties"
-    mode: 0644
-
-- name: Cassandra | Create directories for cassandra
-  become: yes
-  file: 
-    path: "{{ item }}" 
-    owner: cassandra
-    group: cassandra
-    state: directory
-  with_items: "{{ paths }}"
-
-
-- name: Clear cassandra libraries
-  file: path={{ item }} state=absent
-  become: yes
-  with_fileglob:
-    - "{{ cassandra_lib }}/metrics-core-*.jar"
-    - "{{ cassandra_lib }}/reporter-config-base-*.jar"
-    - "{{ cassandra_lib }}/reporter-config3-*.jar"
-
-- name: Unarchive the files
-  unarchive:
-    src: "{{ cassandra_lib_src }}"
-    dest: "{{ cassandra_lib }}"
-  become: yes
-
-- name: Add JAVA_OPTS
-  lineinfile: dest="{{ cassandra_env_file }}"  line='JVM_OPTS="$JVM_OPTS -Dcassandra.metricsReporterConfigFile=graphite.yaml"'
-  become: yes
-
-- name: Cassandra | Restart cassandra
-  become: yes
-  meta: flush_handlers
diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz b/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz
deleted file mode 100644
index 2dcfaed218..0000000000
Binary files a/ansible/roles/cassandra-3.7.0/templates/cassandra-lib.tgz and /dev/null differ
diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2 b/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2
deleted file mode 100644
index 180d1f1904..0000000000
--- a/ansible/roles/cassandra-3.7.0/templates/cassandra-rackdc.properties.j2
+++ /dev/null
@@ -1,27 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# These properties are used with GossipingPropertyFileSnitch and will
-# indicate the rack and dc for this node
-dc=ap-south
-rack=1a
-
-# Add a suffix to a datacenter name. Used by the Ec2Snitch and Ec2MultiRegionSnitch
-# to append a string to the EC2 region name.
-#dc_suffix=
-
-# Uncomment the following line to make this snitch prefer the internal ip when possible, as the Ec2MultiRegionSnitch does.
-# prefer_local=true
diff --git a/ansible/roles/cassandra-3.7.0/templates/cassandra.j2 b/ansible/roles/cassandra-3.7.0/templates/cassandra.j2
deleted file mode 100644
index 7114ea120f..0000000000
--- a/ansible/roles/cassandra-3.7.0/templates/cassandra.j2
+++ /dev/null
@@ -1,180 +0,0 @@
-cluster_name: 'Cassandra Cluster - {{env}}'
-
-num_tokens: 256
-
-hinted_handoff_enabled: true
-
-max_hint_window_in_ms: 10800000
-
-hinted_handoff_throttle_in_kb: 1024
-
-max_hints_delivery_threads: 2
-
-batchlog_replay_throttle_in_kb: 1024
-
-authenticator: AllowAllAuthenticator
-
-authorizer: AllowAllAuthorizer
-
-role_manager: CassandraRoleManager
-
-roles_validity_in_ms: 2000
-
-permissions_validity_in_ms: 2000
-
-partitioner: org.apache.cassandra.dht.Murmur3Partitioner
-
-data_file_directories:
-  - {{data_file_directories}}
-
-commitlog_directory: {{commitlog_directory}}
-
-disk_failure_policy: stop
-
-commit_failure_policy: stop
-
-key_cache_size_in_mb:
-
-key_cache_save_period: 14400
-
-row_cache_size_in_mb: 0
-
-row_cache_save_period: 0
-
-counter_cache_size_in_mb:
-
-counter_cache_save_period: 7200
-
-saved_caches_directory: {{saved_caches_directory}}
-
-commitlog_sync: periodic
-
-commitlog_sync_period_in_ms: 10000
-
-commitlog_segment_size_in_mb: 32
-
-seed_provider:
-  - class_name: org.apache.cassandra.locator.SimpleSeedProvider
-    parameters:
-      - seeds: "{{seeds}}"
-
-concurrent_reads: 32
-
-concurrent_writes: 32
-
-concurrent_counter_writes: 32
-
-concurrent_materialized_view_writes: 32
-
-memtable_allocation_type: heap_buffers
-
-index_summary_capacity_in_mb:
-
-index_summary_resize_interval_in_minutes: 60
-
-trickle_fsync: false
-
-trickle_fsync_interval_in_kb: 10240
-
-storage_port: 7000
-
-ssl_storage_port: 7001
-
-listen_address: {{listen_address}}
-
-start_native_transport: true
-
-native_transport_port: 9042
-
-start_rpc: false
-
-rpc_address: 0.0.0.0
-
-rpc_port: 9160
-
-broadcast_rpc_address: {{broadcast_rpc_address}}
-
-rpc_keepalive: true
-
-rpc_server_type: sync
-
-thrift_framed_transport_size_in_mb: 15
-
-incremental_backups: false
-
-snapshot_before_compaction: false
-
-auto_snapshot: true
-
-tombstone_warn_threshold: 1000
-
-tombstone_failure_threshold: 100000
-
-column_index_size_in_kb: 64
-
-batch_size_warn_threshold_in_kb: 5
-
-batch_size_fail_threshold_in_kb: 50
-
-unlogged_batch_across_partitions_warn_threshold: 10
-
-compaction_throughput_mb_per_sec: 16
-
-compaction_large_partition_warning_threshold_mb: 100
-
-sstable_preemptive_open_interval_in_mb: 50
-
-read_request_timeout_in_ms: 5000
-
-range_request_timeout_in_ms: 10000
-
-write_request_timeout_in_ms: 2000
-
-counter_write_request_timeout_in_ms: 5000
-
-cas_contention_timeout_in_ms: 1000
-
-truncate_request_timeout_in_ms: 60000
-
-request_timeout_in_ms: 10000
-
-cross_node_timeout: false
-
-phi_convict_threshold: 12
-
-endpoint_snitch: GossipingPropertyFileSnitch
-
-dynamic_snitch_update_interval_in_ms: 100
-
-dynamic_snitch_reset_interval_in_ms: 600000
-
-dynamic_snitch_badness_threshold: 0.1
-
-request_scheduler: org.apache.cassandra.scheduler.NoScheduler
-
-server_encryption_options:
-  internode_encryption: none
-  keystore: conf/.keystore
-  keystore_password: cassandra
-  truststore: conf/.truststore
-  truststore_password: cassandra
-
-client_encryption_options:
-  enabled: false
-  optional: false
-  keystore: conf/.keystore
-  keystore_password: cassandra
-
-internode_compression: all
-
-inter_dc_tcp_nodelay: false
-
-tracetype_query_ttl: 86400
-
-tracetype_repair_ttl: 604800
-
-enable_user_defined_functions: false
-
-windows_timer_interval: 1
-
-auto_bootstrap: false
\ No newline at end of file
diff --git a/ansible/roles/cassandra-backup/defaults/main.yml b/ansible/roles/cassandra-backup/defaults/main.yml
index e860f103d4..e47f87ef05 100644
--- a/ansible/roles/cassandra-backup/defaults/main.yml
+++ b/ansible/roles/cassandra-backup/defaults/main.yml
@@ -1,4 +1,5 @@
 cassandra_root_dir: /etc/cassandra
 cassandra_backup_dir: /data/cassandra/backup
-cassandra_backup_azure_container_name: dp-cassandra-backup
 
+cloud_storage_dpcassandrabackup_bucketname: "{{cloud_storage_management_bucketname}}"
+cloud_storage_dpcassandrabackup_foldername: dp-cassandra-backup
diff --git a/ansible/roles/cassandra-backup/meta/main.yml b/ansible/roles/cassandra-backup/meta/main.yml
deleted file mode 100644
index 23b18a800a..0000000000
--- a/ansible/roles/cassandra-backup/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
\ No newline at end of file
diff --git a/ansible/roles/cassandra-backup/tasks/main.yml b/ansible/roles/cassandra-backup/tasks/main.yml
index f65c78cd7a..b4cc6dd8ed 100755
--- a/ansible/roles/cassandra-backup/tasks/main.yml
+++ b/ansible/roles/cassandra-backup/tasks/main.yml
@@ -7,7 +7,6 @@
 - set_fact:
     cassandra_backup_gzip_file_name: "cassandra-backup-{{ lookup('pipe', 'date +%Y%m%d') }}.tar.gz"
 
- 
 - set_fact:
     cassandra_backup_gzip_file_path: "{{ cassandra_backup_dir }}/{{ cassandra_backup_gzip_file_name }}.tar.gz"
 
@@ -25,14 +24,44 @@
 - name: print doc_root to console
   debug:
     var: doc_data    
-
-- name: upload to azure
+    
+- name: upload file to azure storage using azcopy
   include_role:
-    name: artifacts-upload-azure
+    name: azure-cloud-storage
+    tasks_from: upload-using-azcopy.yml
   vars:
-    artifact: "{{ cassandra_backup_gzip_file_name }}"
-    artifact_path: "{{ cassandra_backup_gzip_file_path }}"
-    artifacts_container: "{{ cassandra_backup_azure_container_name }}"
+    blob_container_name: "{{ cloud_storage_dpcassandrabackup_foldername }}"
+    container_public_access: "off"
+    blob_container_folder_path: ""
+    local_file_or_folder_path: "{{ cassandra_backup_gzip_file_path }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
    
+- name: upload backup to S3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: upload-folder.yml
+  vars:
+    local_file_or_folder_path: "{{ cassandra_backup_gzip_file_path }}"
+    s3_bucket_name: "{{ cloud_storage_dpcassandrabackup_bucketname }}"
+    s3_path: "{{ cloud_storage_dpcassandrabackup_foldername }}/{{ cassandra_backup_gzip_file_name}}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "aws"
+  
+- name: upload file to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload-batch.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dpcassandrabackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpcassandrabackup_foldername }}/{{ cassandra_backup_gzip_file_name}}"
+    local_file_or_folder_path: "{{ cassandra_backup_gzip_file_path }}"
+  when: cloud_service_provider == "gcloud"
+  
 - name: clean up backup dir after upload
   file: path={{ cassandra_backup_dir }} state=absent
diff --git a/ansible/roles/cassandra-restore/defaults/main.yml b/ansible/roles/cassandra-restore/defaults/main.yml
index a58c0a40b2..658def4ea8 100644
--- a/ansible/roles/cassandra-restore/defaults/main.yml
+++ b/ansible/roles/cassandra-restore/defaults/main.yml
@@ -1 +1,4 @@
-user_home: /home/deployer
\ No newline at end of file
+user_home: "/home/{{ ansible_ssh_user }}/"
+
+cloud_storage_dpcassandrabackup_bucketname: "{{cloud_storage_management_bucketname}}"
+cloud_storage_dpcassandrabackup_foldername: 'cassandra-backup'
diff --git a/ansible/roles/cassandra-restore/meta/main.yml b/ansible/roles/cassandra-restore/meta/main.yml
deleted file mode 100644
index 23b18a800a..0000000000
--- a/ansible/roles/cassandra-restore/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
\ No newline at end of file
diff --git a/ansible/roles/cassandra-restore/tasks/main.yml b/ansible/roles/cassandra-restore/tasks/main.yml
index 8f54faf0c8..c9dcceb130 100755
--- a/ansible/roles/cassandra-restore/tasks/main.yml
+++ b/ansible/roles/cassandra-restore/tasks/main.yml
@@ -3,12 +3,44 @@
 
 - set_fact:
     artifact_path: "/tmp/{{ artifact }}"
-    artifacts_container: "{{ cassandra_backup_azure_container_name }}"
     cassandra_restore_dir: /tmp/cassandra_backup
 
-- name: download from azure
+- name: download a file from azure storage
+  become: true
   include_role:
-    name: artifacts-download-azure
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dpcassandrabackup_foldername }}"
+    blob_file_name: "{{ artifact }}"
+    local_file_or_folder_path: "{{ artifact_path }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+
+- name: download a file from aws s3
+  become: true
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dpcassandrabackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ artifact_path  }}"
+    s3_path: "{{ cloud_storage_dpcassandrabackup_foldername }}/{{ artifact }}"
+  when: cloud_service_provider == "aws"
+
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_bucket_name: "{{ cloud_storage_dpcassandrabackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpcassandrabackup_foldername }}/{{ artifact }}"
+    local_file_or_folder_path: "{{ artifact_path }}"
+  when: cloud_service_provider == "gcloud"
 
 - name: extract the archive
   unarchive: src=/tmp/{{ artifact }} dest=/tmp remote_src=yes
diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml
index c38ddf411a..e969beb01a 100644
--- a/ansible/roles/content-snapshot-indexer/defaults/main.yml
+++ b/ansible/roles/content-snapshot-indexer/defaults/main.yml
@@ -2,14 +2,19 @@ analytics_user: analytics
 analytics_group: analytics
 content_snapshot_path: "/mount/data/analytics/content-snapshot"
 content_snapshot_jar_path: "{{content_snapshot_path}}/jars" # Add content snapshot jar file path.
-content_snapshot_jar_name: "adhoc-jobs-1.0.jar"
-content_snapshot_distribuction_name: "adhoc-jobs-1.0-distribution.tar.gz"
-spark_home: "/mount/data/analytics/spark-2.4.4-bin-hadoop2.7"
+content_snapshot_jar_name: "etl-jobs-1.0.jar"
+content_snapshot_distribuction_name: "etl-jobs-1.0-distribution.tar.gz"
+
+cloud_storage_url: "https://bmzbbujw9kal.compat.objectstorage.ap-mumbai-1.oraclecloud.com"
+sunbird_public_storage_account_endpoint: "{{cloud_storage_url}}"
+
+spark_home: "/mount/data/analytics/spark-3.1.3-bin-hadoop2.7/"
 redis:
-  host: "{{metadata_redis_host}}" # Redis DP IP 
+  host: "{{metadata2_redis_host}}" # Redis DP IP
   port: "6379"
   user:
-    index: "4"
+    index: "12" # For temporary will change once verification is done
+    input_index: "4" # Input index for string format
     source_key: "id"
   content:
     index: "5"
@@ -23,8 +28,9 @@ redis:
     min_evict_idle_seconds: 120
     time_evict_run_second: 300
     max: 20
-  max_pipeline_size: "1000"
+  max_pipeline_size: "100000"
   key_expiry_seconds: 3600
+  scan_count: "100000"
 
 elastic_search:
   host: "{{ lp_composite_search_host }}" # compositesearch index
@@ -41,15 +47,21 @@ elastic_search:
   scroll_size: "1000"
 
 cloud_storage:
-  container: "telemetry-data-store" # Container is different in all env so override this.
-  object_key: "druid-content-snapshot/snapshot.txt"
-  provider: "azure"
-  account_name: "{{ azure_storage_account }}"
-  account_key: "{{ azure_storage_secret }}"
+  container: "odev-dev-diksha-telemetry" # Container is different in all env so override this.
+  #object_key: "druid-content-snapshot/snapshot.txt"
+  object_key: "vdn_content_model_snapshot/vdn-snapshot.txt"
+  provider: "oci"
+  account_name: "{{sunbird_public_storage_account_name}}"
+  account_key: "{{sunbird_public_storage_account_key}}"
+  #account_endpoint: "{{sunbird_public_storage_account_endpoint}}"
+  account_endpoint: "https://bmzbbujw9kal.compat.objectstorage.ap-mumbai-1.oraclecloud.com"
 
 cassandra:
-  host: "{{groups['lp-cassandra'][0]}}" ## LMS-Cassandra IP Address.
-  keyspace: 'sunbird'
+  host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address.
+  keyspace: "sunbird"
+  read_timeout: "500000"
+  retry_count: "100"
+  input_consistency_level: "LOCAL_QUORUM"
   user:
     table: "user"
   device_profile:
@@ -57,9 +69,9 @@ cassandra:
     new_table: "device_profile_temp"
 
 druid:
-  coordinator_host: "{{groups['coordinator'][0]}}"
-  data_source: "content-model-snapshot"
-  ingestion_spec_path: "{{ content_snapshot_jar_path }}/adhoc-jobs-1.0/druid_models/content_index_batch.json"
+  coordinator_host: "{{ groups['rollup-coordinator'][0] | default(groups['raw-coordinator'][0]) }}" 
+  data_source: "vdn_content_model_snapshot"
+  ingestion_spec_path: "{{ content_snapshot_jar_path }}/etl-jobs-1.0/druid_models/vdn_content_index_batch.json"
 
 job_config:
   es_cloud_uploader_path: "{{ content_snapshot_path }}/config/ESCloudUploader.conf"
diff --git a/ansible/roles/content-snapshot-indexer/tasks/main.yml b/ansible/roles/content-snapshot-indexer/tasks/main.yml
index fefb9ce0f2..bbee7786bc 100644
--- a/ansible/roles/content-snapshot-indexer/tasks/main.yml
+++ b/ansible/roles/content-snapshot-indexer/tasks/main.yml
@@ -47,13 +47,13 @@
   tags:
     - deploy
 
-- name: Unarchive adhoc job script
+- name: Unarchive etl job script
   become: yes
   unarchive: src={{ content_snapshot_path }}/{{ content_snapshot_distribuction_name }} dest={{ content_snapshot_jar_path }} copy=no group={{ analytics_group }} owner={{ analytics_user }}
   tags:
     - deploy
 
-- name: Copy adhoc-jobs jar file
+- name: Copy etl-jobs jar file
   become: yes
   copy:
     src: "{{ content_snapshot_jar_name }}"
@@ -69,7 +69,7 @@
     path: '{{ content_snapshot_path }}/config'
     state: absent
   tags:
-    - deploy
+    - execute-script
 
 - name: Create config directory
   file:
@@ -77,7 +77,7 @@
     state: directory
   become: yes
   tags:
-    - deploy
+    - execute-script
 
 - name: Ensure copying all conf files from the templates/conf folder
   become: yes
@@ -88,7 +88,7 @@
     group: "{{ analytics_group }}"
   with_items: "{{ config_files }}"
   tags:
-    - deploy
+    - execute-script
 
 - name: Ensure removing scripts directory
   become: yes
@@ -97,7 +97,7 @@
     state: absent
   become_user: "{{ analytics_user }}"  
   tags:
-    - deploy
+    - execute-script
 
 - name: Create scripts directory
   become: yes
@@ -106,7 +106,7 @@
     state: directory
   become: yes
   tags:
-    - deploy
+    - execute-script
 
 - name: Ensure copying all script files from the templates/script dir
   become: yes
@@ -118,12 +118,12 @@
     mode: 0777
   with_items: "{{ script_files }}"
   tags:
-    - deploy
+    - execute-script
 
 - name: Run Script
   become: yes
   become_user: root
-  shell: "sh {{ content_snapshot_path }}/scripts/run-script.sh {{ script_to_run }} >> {{ content_snapshot_path }}/logs/{{ansible_date_time.date}}-shell-execution.log"
+  shell: "sh {{ content_snapshot_path }}/scripts/run-script.sh {{ script_to_run }} {{ identifier }} {{ date }} {{ populate_anonymous_user }} {{ refresh_data }} >> {{ content_snapshot_path }}/logs/{{ansible_date_time.date}}-shell-execution.log"
   poll: 0
   async: 45
   tags:
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
index 7826108361..337759f5d1 100644
--- a/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/DialcodeRedisIndexer.j2
@@ -1,6 +1,6 @@
 # redis
 redis.host="{{ redis.host }}"
-redis.port="{{ redis.port }}"
+redis.port="{{ dialcode_port }}"
 redis.connection.max={{ redis.connection.max }}
 redis.connection.idle.max={{ redis.connection.idle_max }}
 redis.connection.idle.min={{ redis.connection.idle_min }}
@@ -10,7 +10,7 @@ redis.max.pipeline.size="{{ redis.max_pipeline_size }}"
 
 redis.dialcode.database.index=6
 
-cloudStorage.accountName="{{ azure_storage_account }}"
-cloudStorage.accountKey="{{ azure_storage_secret }}"
-cloudStorage.container="{{ bucket }}"
-cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv"
\ No newline at end of file
+cloudStorage.accountName="{{sunbird_private_storage_account_name}}"
+cloudStorage.accountKey="{{sunbird_private_storage_account_key}}"
+cloudStorage.container="{{ bucket | default('telemetry-data-store') }}"
+cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv"
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
index 62312f203f..dab231923f 100755
--- a/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/ESCloudUploader.j2
@@ -8,4 +8,5 @@ cloudStorage.container="{{ cloud_storage.container }}"
 cloudStorage.objectKey="{{ cloud_storage.object_key }}"
 cloudStorage.provider="{{ cloud_storage.provider }}"
 cloudStorage.accountName="{{ cloud_storage.account_name }}"
-cloudStorage.accountKey="{{ cloud_storage.account_key }}"
\ No newline at end of file
+cloudStorage.accountKey="{{ cloud_storage.account_key }}"
+cloudStorage.accountEndpoint="{{ cloud_storage.account_endpoint }}"
\ No newline at end of file
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2
index 52b7ea2b3f..724051d23b 100755
--- a/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/ESContentIndexer.j2
@@ -1,6 +1,6 @@
 # redis
 redis.host="{{ redis.host }}"
-redis.port="{{ redis.port }}"
+redis.port="{{ content_port }}"
 redis.connection.max={{ redis.connection.max }}
 location.db.redis.key.expiry.seconds={{ redis.key_expiry_seconds }}
 redis.connection.idle.max={{ redis.connection.idle_max }}
diff --git a/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2 b/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2
index 12db51e672..47681b82c5 100755
--- a/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/conf/cassandraRedis.j2
@@ -1,6 +1,6 @@
 # redis
 redis.host="{{ redis.host }}"
-redis.port="{{redis.port }}"
+redis.port="{{ user_port }}"
 redis.connection.max={{ redis.connection.max }}
 location.db.redis.key.expiry.seconds={{ redis.key_expiry_seconds }}
 redis.connection.idle.max={{ redis.connection.idle_max }}
@@ -15,4 +15,10 @@ spark.cassandra.connection.host="{{ cassandra.host }}"
 cassandra.user.keyspace="{{ cassandra.keyspace }}"
 cassandra.user.table="{{ cassandra.user.table }}"
 redis.user.database.index="{{ redis.user.index }}"
+redis.user.input.index="{{ redis.user.input_index }}"
+redis.user.backup.dir="/mount/data/analytics/content-snapshot/anonymous-user-backup"
+redis.scan.count="{{ redis.scan_count }}"
 redis.user.index.source.key="{{ redis.user.source_key }}" # this will be used as key for redis
+cassandra.read.timeoutMS="{{ cassandra.read_timeout }}"
+cassandra.query.retry.count="{{ cassandra.retry_count }}"
+cassandra.input.consistency.level="{{ cassandra.input_consistency_level }}"
\ No newline at end of file
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2
index 1da516c8bf..b05762a442 100644
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/DeviceProfileScripts.j2
@@ -7,7 +7,7 @@ today=$(date "+%Y-%m-%d")
 
 nohup {{ spark_home }}/bin/spark-submit \
 --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.DeviceProfileUpdateCassandra \
+--class org.sunbird.analytics.jobs.DeviceProfileUpdateCassandra \
 ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
 
 
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2
index 91ca9ba60a..20a6d5ae02 100644
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/DruidContentIndexer.j2
@@ -10,7 +10,7 @@ home=`echo $HOME`
 ingestionSpecFilePath="{{ druid.ingestion_spec_path }}"
 jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}"
 jobConfPath="{{ job_config.es_cloud_uploader_path }}"
-libs_path="{{ content_snapshot_path }}/jars/adhoc-jobs-1.0"
+libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0"
 export SPARK_HOME="{{ spark_home }}"
 
 # get list of segments from content model snapshot datasource
@@ -28,7 +28,7 @@ echo "STARTED EXECUTING USER DRUID CONTENT INDEXER..."
 #--class org.ekstep.analytics.jobs.ESCloudUploader \
 #${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
 
-nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}"  --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',')  --class org.ekstep.analytics.jobs.ESCloudUploader ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
+nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}"  --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',')  --class org.sunbird.analytics.jobs.ESCloudUploader ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
 
 printf "\n>>> submit ingestion task to Druid!\n"
 
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2
index 7f86ec9c58..95db9071cd 100644
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisContentIndexer.j2
@@ -6,10 +6,12 @@ home=`echo $HOME`
 jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}"
 jobConfPath="{{ job_config.es_content_indexer_path }}"
 today=$(date "+%Y-%m-%d")
+libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0"
 
 echo "STARTED EXECUTING CONTENT CACHE INDEXER..."
 
 nohup {{ spark_home }}/bin/spark-submit \
 --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.ESRedisIndexer \
+--class org.sunbird.analytics.jobs.ESRedisIndexer \
+--jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') \
 ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2
index 8fe8850554..98f4144af1 100755
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisDialcodeIndexer.j2
@@ -4,7 +4,7 @@ home=`echo $HOME`
 jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}"
 jobConfPath="{{ job_config.es_dialCode_indexer_path }}"
 today=$(date "+%Y-%m-%d")
-libs_path="{{ content_snapshot_path }}/jars/adhoc-jobs-1.0"
+libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0"
 export SPARK_HOME="{{ spark_home }}"
 
 echo "STARTED EXECUTING DIALCODE CACHE INDEXER..."
@@ -14,4 +14,4 @@ echo "STARTED EXECUTING DIALCODE CACHE INDEXER..."
 #--class org.ekstep.analytics.jobs.CSVToRedisIndexer \
 #${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
 
-nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}"  --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',')  --class org.ekstep.analytics.jobs.CSVToRedisIndexer ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
+nohup $SPARK_HOME/bin/spark-submit --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}"  --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',')  --class org.sunbird.analytics.jobs.CSVToRedisIndexer ${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2
index bc7b5d1d81..5810dfeca1 100644
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/RedisUserDataIndexer.j2
@@ -1,13 +1,20 @@
 #!/usr/bin/env bash
 
+#$1 identifier: userid- Especially for UserCache indexer script to cache specific user id details
+#$2 date: YYYY-MM-DD- Especially for UserCache indexer script to cache all the user data from this specific updated date.
+#$3 populate_anonymous_user: true/false - Especially for UserCacheIndexer to read anonymous user's data
+#$4 refresh_user_data: true/false - Especially for UserCacheIndexer to refresh user's data of redis
+
 home=`echo $HOME`
 jobJarPath="{{ content_snapshot_jar_path }}/{{ content_snapshot_jar_name }}"
 jobConfPath="{{ job_config.cassandra_redis_path }}"
 today=$(date "+%Y-%m-%d")
+libs_path="{{ content_snapshot_path }}/jars/etl-jobs-1.0"
 
-echo "STARTED EXECUTING USER CACHE INDEXER..."
+echo "STARTED EXECUTING USER CACHE INDEXER... $1 $2 $3 $4"
 
 nohup {{ spark_home }}/bin/spark-submit \
 --conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.CassandraRedisIndexer \
-${jobJarPath} >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
+--jars $(echo ${libs_path}/lib/*.jar | tr ' ' ',') \
+--class org.sunbird.analytics.jobs.UserCacheIndexer \
+${jobJarPath} $1 $2 $3 $4 >> "{{ content_snapshot_path }}/logs/$today-task-execution.log" 2>&1
\ No newline at end of file
diff --git a/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2 b/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2
index 7d274d0ef7..c4b2cc46b1 100644
--- a/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2
+++ b/ansible/roles/content-snapshot-indexer/templates/scripts/run-script.j2
@@ -1,10 +1,16 @@
 #!/usr/bin/env bash
 
-echo "Executing the script $1"
+#$1 script_to_run: It can be USER_CACHE_INDEXER, DRUID_CONTENT_INDEXER etc,.
+#$2 identifier: userid- Especially for UserCache indexer script to cache specific user id details
+#$3 date: YYYY-MM-DD- Especially for UserCache indexer script to cache all the user data from this specific updated date.
+#$4 populate_anonymous_user: true/false - Especially for UserCacheIndexer to read anonymous user's data
+#$5 refresh_user_data: true/false - Especially for UserCacheIndexer to refresh user's data of redis
+
+echo "Executing the script $1 $2 $3 $4 $5"
 case "$1" in
    "USER_CACHE_INDEXER") 
       echo "Invoked RedisUserDataIndexer"
-      nohup {{ content_snapshot_path }}/scripts/RedisUserDataIndexer.sh & 
+      nohup {{ content_snapshot_path }}/scripts/RedisUserDataIndexer.sh $2 $3 $4 $5 &
    ;;
    "DRUID_CONTENT_INDEXER")
       echo "Invoked DruidContentIndexer"
diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml
old mode 100644
new mode 100755
index 9ac351d19d..690c51d87d
--- a/ansible/roles/data-products-deploy/defaults/main.yml
+++ b/ansible/roles/data-products-deploy/defaults/main.yml
@@ -1,45 +1,53 @@
 analytics_user: analytics
 analytics_group: analytics
 spark_output_temp_dir: /mount/data/analytics/tmp/
-telemetry_converter_ver: 0.0.8
 
-bucket: "dev-data-store"
+bucket: "telemetry-data-store"
+secor_bucket: "telemetry-data-store"
+dp_object_store_type: "azure"
+dp_raw_telemetry_backup_location: "unique/raw/"
+dp_storage_key_config: "azure_storage_key"
+dp_storage_secret_config: "azure_storage_secret"
+dp_reports_storage_key_config: "reports_azure_storage_key"
+dp_reports_storage_secret_config: "reports_azure_storage_secret"
+
 kafka_broker_host: "{{groups['processing-cluster-kafka'][0]}}:9092"
+ingestion_kafka_broker_host: "{{groups['ingestion-cluster-kafka'][0]}}:9092"
 brokerlist: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
 zookeeper: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
 dp_username: dp-monitor
 analytics_job_queue_topic: "{{ env }}.analytics.job_queue"
-datasetRawBucket: ekstep-data-sets-{{ env }}
-dataExhaustBucket: ekstep-public-{{ env }}
-dataExhaustPrefix: data-exhaust/
-consumptionRawPrefix: datasets/D001/4208ab995984d222b59299e5103d350a842d8d41/
 topic: "{{ env }}.telemetry.derived"
-learning_topic: "{{ env }}.learning.graph.events"
 analytics_metrics_topic: "{{ env }}.analytics_metrics"
 sink_topic: "{{ env }}.telemetry.sink"
+assess_topic: "{{ env }}.telemetry.assess"
 metrics_topic: "{{ env }}.telemetry.metrics"
 job_manager_tmp_dir: "transient-data"
 channel: dev-test
 druid_broker_host: "{{groups['raw-broker'][0]}}"
+druid_router_host: "{{groups['raw-router'][0]}}"
 druid_rollup_broker_host: "{{groups['raw-broker'][0]}}"
 hierarchySearchServiceUrl: "{{ proto }}://{{ domain_name }}/action/content"
 hierarchySearchServicEndpoint: /v3/hierarchy/
 
-analytics_job_list: '"wfs", "ds", "dpu", "content-rating-updater", "audit-metrics-report", "monitor-job-summ"'
-analytics_jobs_count: 8
+user_table_keyspace: "sunbird"
+course_keyspace: "sunbird_courses"
+hierarchy_store_keyspace: "{{ env }}_hierarchy_store"
+job_request_table: "{{ env }}_job_request"
+dataset_metadata_table: "{{ env }}_dataset_metadata"
+report_user_table_keyspace: "sunbird_courses"
+report_user_enrolment_table: "report_user_enrolments"
+
+analytics_job_list: '"wfs", "content-rating-updater", "monitor-job-summ"'
+analytics_jobs_count: 3
 
 cassandra_keyspace_prefix: '{{ env }}_'
-dp_dispatch_bucket_name: ekstep-dev-data-store
-data_exhaust_blob_name: sunbird1p7
-data_exhaust_s3_url: https://s3-ap-south-1.amazonaws.com
-spark_version: 2.4.4
+report_cassandra_cluster_host: "{{ report_cassandra_host | default(core_cassandra_host) }}"
+cassandra_hierarchy_store_keyspace: "{{ env_name}}_hierarchy_store"
+spark_version: 3.1.3
 
 heap_memory: "-Xmx5120m"
 
-business_metrics:
-  s3_path: "s3://ekstep-{{ env }}-data-store/business_metrics/"
-  env: "{{ env }}"
-
 spark:
   home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7"
   public_dns: 54.255.154.146
@@ -62,16 +70,16 @@ spark:
 
 submit_jobs:
   submit-all-jobs:
-    hour: 00
+    hour: 02
     minute: 35
 
 start_jobmanager:
   job-manager:
-    hour: 00
+    hour: 02
     minute: 30
 have_weekly_jobs: false
 
-video_stream_job_schedule: 10
+course_batch_status_updater_job_schedule: 60
 
 run_wfs_job:
   wfs:
@@ -82,16 +90,6 @@ run_monitor_job:
     hour: 03
     minute: 00
 
-run_course_metrics_job:
-  course-dashboard-metrics:
-    hour: 19
-    minute: 45
-
-run_assessment_metrics_job:
-  assessment-dashboard-metrics:
-    hour: 20
-    minute: 15
-
 run_admin_user_reports_job:
   admin-user-reports-3AMIST:
     hour: 21
@@ -123,24 +121,55 @@ index_content_model_druid:
     hour: 1
     minute: 00
 
-run_district_weekly_job:
-  district-weekly:
-    hour: 03
-    minute: 30
-    weekday: 1
-
 run_etb_metrics_weekly_job:
   etb-metrics-weekly:
     hour: 23
     minute: 30
     weekday: 1
 
-run_district_monthly_job:
-  district-monthly:
-    hour: 04
+# These are the dummy times till sept30 for exhaust reports
+#To-Do: Update time after 3.2.7 deployment
+
+run_progress_exhaust:
+  progress-exhaust:
+    hour: 08
+    minute: 00
+
+run_response_exhaust:
+  response-exhaust:
+    hour: 09
+    minute: 00
+
+run_userinfo_exhaust:
+  userinfo-exhaust:
+    hour: 10
+    minute: 00
+
+run_collection_summary:
+  collection-summary:
+    hour: 09
+    minute: 30
+
+run_sourcing_summary:
+  sourcing-summary:
+    hour: 10
     minute: 30
-    day: 1
-    month: 1-12         
+
+run_cassandra_migration:
+  cassandra-migration:
+    hour: 19
+    minute: 15
+
+run_uci_private_exhaust_job:
+  uci-private-exhaust:
+    hour: 03
+    minute: 00 
+
+run_uci_response_exhaust_job:
+  uci-response-exhaust:
+    hour: 02
+    minute: 00
+    
 
 service:
   search:
@@ -151,19 +180,103 @@ es_search_index: "compositesearch"
 analytics:
   home: /mount/data/analytics
   soft_path: /mount/data/analytics
-  paths: ['/home/analytics/sbin', '/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/logs/api-service', '/mount/data/analytics/api', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts/monitor-data', '/mount/data/analytics/models' ]
-  scripts: ['model-config', 'replay-job', 'replay-updater', 'replay-utils', 'run-job', 'monitor-dp', 'generate-metrics', 'submit-job', 'start-jobmanager']
+  paths: ['/mount/data/analytics', '/mount/data/analytics/logs', '/mount/data/analytics/logs/services', '/mount/data/analytics/logs/data-products', '/mount/data/analytics/tmp', '/mount/data/analytics/scripts', '/mount/data/analytics/models' ]
+  scripts: ['model-config', 'replay-job', 'replay-updater', 'replay-utils', 'run-job', 'submit-job', 'start-jobmanager', 'submit-script']
+  dockScripts: ['model-dock-config','run-dock-job']
 
+# artifact versions
+analytics_core_artifact_ver: "2.0"
+analytics_ed_dataporducts_artifact_ver: "1.0"
+scruid_artifact_ver: "2.5.0"
 
 producer_env: "dev.sunbird"
-analytics_job_manager_artifact: "job-manager-2.0.jar"
-analytics_core_artifact: "analytics-framework-2.0.jar"
-scruid_artifact: "scruid_2.11-2.3.2.jar"
-analytics_batch_module_artifact: "batch-models-2.0.jar"
-analytics_ed_dataporducts_artifact: "data-products-1.0-distribution.tar.gz"
+analytics_job_manager_artifact: "job-manager-{{ analytics_core_artifact_ver }}.jar"
+analytics_core_artifact: "analytics-framework-{{ analytics_core_artifact_ver }}.jar"
+scruid_artifact: "scruid_2.12-{{ scruid_artifact_ver }}.jar"
+analytics_batch_module_artifact: "batch-models-{{ analytics_core_artifact_ver }}.jar"
+analytics_ed_dataporducts_artifact: "data-products-{{ analytics_ed_dataporducts_artifact_ver }}-distribution.tar.gz"
 model_version: "2.0" 
 
-submit_jobs_auth_token: ""
-report_list_jobs_url: "http://{{groups['analytics-api'][0]}}:9000/"
+submit_jobs_auth_token: "{{ sunbird_api_auth_token }}"
+report_list_jobs_url: "{{ druid_report_url }}"
 
 reports_container: "reports"
+
+# Cluster vars
+spark_cluster_user_password: ""
+spark_cluster_user_name: ""
+admin_name: "{{ spark_cluster_user_name }}"
+admin_password: "{{ spark_cluster_user_password }}"
+spark_cluster_name: "{{env}}-spark-cluster"
+
+spark_cluster:
+  executor_core: 5
+  executor_memory: 19G
+  num_executors: 5
+
+analytics_cluster:
+  home: "/tmp"
+
+analytics_ed_dataporducts_jar_artifact: "data-products-{{ analytics_ed_dataporducts_artifact_ver }}.jar"  
+
+spark_enable_dynamic_allocation: false
+# Spark Cassandra config-vars
+spark_cassandra_connection_timeout_millis: 30000
+spark_cassandra_query_timeout_millis: 180000
+spark_cassandra_query_max_rows_fetch_count: 1000
+spark_sql_shuffle_partitions: 200
+
+druid_report_postgres_db_name: druid
+druid_report_postgres_db_username: druid
+
+
+#Override this variable in production and point to druid rollup ingestion cluster 
+# Example: "http://$rollup_cluster_ip:8090"
+druid_rollup_cluster_ingestion_task_url: "http://{{groups['raw-overlord'][0]}}:8081"
+
+# On demand Exhaust throttling vars
+exhaust_batches_limit_per_channel: 30
+exhaust_file_size_limit_bytes_per_channel: 1073741824
+
+exhaust_parallel_batch_load_limit: 10
+exhaust_user_parallelism: 200
+
+data_exhaust_batch_limit_per_request: 20
+
+# Start Of UCI Related Variables
+uci_postgres_host: "dev-pg11.postgres.database.azure.com"
+uci_encryption_key_base64: ""
+uci_bot_postgres_database: uci-botdb
+uci_fusionauth_postgres_database: uci-fusionauth
+uci_postgres_user: "{{postgres.db_username}}"
+uci_postgres_password: "{{postgres.db_password}}"
+
+uci_postgres:
+  conversation_db_name: "{{ uci_bot_postgres_database }}"
+  conversation_db_host: "{{ uci_postgres_host }}"
+  conversation_db_port: "5432"
+  conversation_db_user: "{{ uci_postgres_user }}"
+  conversation_db_psss: "{{ uci_postgres_password }}"
+  conversation_table_name: "bot"
+  fushionauth_db_name: "{{ uci_fusionauth_postgres_database }}"
+  fushionauth_db_host: "{{ uci_postgres_host }}"
+  fushionauth_db_port: "5432"
+  fushionauth_db_user: "{{ uci_postgres_user }}"
+  fushionauth_db_psss: "{{ uci_postgres_password }}"
+  user_table_name: "users"
+  user_registration_table_name: "user_registrations"
+  user_identities_table_name: "identities"
+
+uci_encryption_secret_key: "{{uci_encryption_key_base64}}"
+uci_pdata_id: "{{uci_env}}.uci.{{sunbird_instance}}"
+
+# End Of UCI Related Variables 
+
+# Exhaust sanity check vars
+cassandra_migrator_job_name: "Cassandra Migrator"
+
+assessment_metric_primary_category: "{{ exhaust_job_assessment_primary_category }}"
+
+# Default s3 variables
+sunbird_private_s3_storage_key: ""
+sunbird_private_s3_storage_secret: ""
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json b/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json
new file mode 100644
index 0000000000..69e13196e2
--- /dev/null
+++ b/ansible/roles/data-products-deploy/files/collection-summary-ingestion-spec.json
@@ -0,0 +1,251 @@
+{
+  "type": "index",
+  "spec": {
+    "dataSchema": {
+      "dataSource": "collection-summary-snapshot",
+      "parser": {
+        "type": "string",
+        "parseSpec": {
+          "format": "json",
+          "flattenSpec": {
+            "useFieldDiscovery": false,
+            "fields": [
+              {
+                "type": "root",
+                "name": "content_org",
+                "expr": "contentorg"
+              },
+              {
+                "type": "root",
+                "name": "user_org",
+                "expr": "orgname"
+              },
+              {
+                "type": "root",
+                "name": "batch_start_date",
+                "expr": "startdate"
+              },
+              {
+                "type": "root",
+                "name": "batch_end_date",
+                "expr": "enddate"
+              },
+              {
+                "type": "root",
+                "name": "has_certificate",
+                "expr": "hascertified"
+              },
+              {
+                "type": "root",
+                "name": "collection_id",
+                "expr": "courseid"
+              },
+              {
+                "type": "root",
+                "name": "batch_id",
+                "expr": "batchid"
+              },
+              {
+                "type": "root",
+                "name": "collection_name",
+                "expr": "collectionname"
+              },
+              {
+                "type": "root",
+                "name": "batch_name",
+                "expr": "batchname"
+              },
+              {
+                "type": "root",
+                "name": "total_enrolment",
+                "expr": "enrolleduserscount"
+              },
+              {
+                "type": "root",
+                "name": "total_completion",
+                "expr": "completionuserscount"
+              },
+              {
+                "type": "root",
+                "name": "total_certificates_issued",
+                "expr": "certificateissuedcount"
+              },
+              {
+                "type": "root",
+                "name": "content_status",
+                "expr": "contentstatus"
+              },
+              {
+                "type": "root",
+                "name": "user_state",
+                "expr": "state"
+              },
+              {
+                "type": "root",
+                "name": "user_district",
+                "expr": "district"
+              },
+              {
+                "type": "root",
+                "name": "content_channel",
+                "expr": "channel"
+              },
+              {
+                "type": "root",
+                "name": "keywords",
+                "expr": "keywords"
+              },
+              {
+                "type": "root",
+                "name": "timestamp",
+                "expr": "timestamp"
+              },
+              {
+                "type": "root",
+                "name": "medium",
+                "expr": "medium"
+              },
+              {
+                "type": "root",
+                "name": "subject",
+                "expr": "subject"
+              },
+              {
+                "type": "root",
+                "name": "created_for",
+                "expr": "createdfor"
+              },
+              {
+                "type": "root",
+                "name": "user_type",
+                "expr": "usertype"
+              },
+              {
+                "type": "root",
+                "name": "user_subtype",
+                "expr": "usersubtype"
+              }
+            ]
+          },
+          "dimensionsSpec": {
+            "dimensions": [
+              {
+                "name": "content_org"
+              },
+              {
+                "name": "user_org"
+              },
+              {
+                "type": "string",
+                "name": "batch_id"
+              },
+              {
+                "type": "string",
+                "name": "batch_start_date"
+              },
+              {
+                "type": "string",
+                "name": "batch_end_date"
+              },
+              {
+                "type": "string",
+                "name": "collection_id"
+              },
+              {
+                "type": "string",
+                "name": "collection_name"
+              },
+              {
+                "type": "string",
+                "name": "batch_name"
+              },
+              {
+                "type": "long",
+                "name": "total_enrolment"
+              },
+              {
+                "type": "long",
+                "name": "total_completion"
+              },
+              {
+                "type": "long",
+                "name": "total_certificates_issued"
+              },
+              {
+                "type": "string",
+                "name": "content_status"
+              },
+              {
+                "type": "string",
+                "name": "user_state"
+              },
+              {
+                "type": "string",
+                "name": "user_district"
+              },
+              {
+                "name": "keywords"
+              },
+              {
+                "name": "has_certificate"
+              },
+              {
+                "type": "string",
+                "name": "content_channel"
+              },
+              {
+                "name": "medium"
+              },
+              {
+                "name": "subject"
+              },
+              {
+                "name": "created_for"
+              },
+              {
+                "type": "string",
+                "name": "user_type"
+              },
+              {
+                "type": "string",
+                "name": "user_subtype"
+              }
+            ],
+            "dimensionsExclusions": []
+          },
+          "timestampSpec": {
+            "column": "timestamp",
+            "format": "auto"
+          }
+        }
+      },
+      "metricsSpec": [],
+      "granularitySpec": {
+        "type": "uniform",
+        "segmentGranularity": "day",
+        "queryGranularity": "none",
+        "rollup": true
+      }
+    },
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "reports",
+            "path": "/collection-summary-reports-v2/collection-summary-report-latest.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "targetPartitionSize": 5000000,
+      "maxRowsInMemory": 25000,
+      "forceExtendableShardSpecs": false,
+      "logParseExceptions": true
+    }
+  }
+}
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json b/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json
new file mode 100644
index 0000000000..69e773d457
--- /dev/null
+++ b/ansible/roles/data-products-deploy/files/sourcing-ingestion-spec.json
@@ -0,0 +1,146 @@
+{
+    "type": "index",
+    "spec": {
+      "dataSchema": {
+        "dataSource": "sourcing-summary-snapshot",
+        "parser": {
+          "type": "string",
+          "parseSpec": {
+            "format": "json",
+            "flattenSpec": {
+              "useFieldDiscovery": false,
+              "fields": [
+                {
+                  "type": "root",
+                  "name": "program_id",
+                  "expr": "program_id"
+                },
+                {
+                  "type": "root",
+                  "name": "status",
+                  "expr": "status"
+                },
+                {
+                  "type": "root",
+                  "name": "rootorg_id",
+                  "expr": "rootorg_id"
+                },
+                {
+                  "type": "root",
+                  "name": "user_id",
+                  "expr": "user_id"
+                },
+                {
+                  "type": "root",
+                  "name": "osid",
+                  "expr": "osid"
+                },
+                {
+                  "type": "root",
+                  "name": "user_type",
+                  "expr": "user_type"
+                },
+                {
+                  "type": "root",
+                  "name": "contributor_id",
+                  "expr": "contributor_id"
+                },
+                {
+                  "type": "root",
+                  "name": "total_contributed_content",
+                  "expr": "total_contributed_content"
+                },
+                {
+                  "type": "root",
+                  "name": "primary_category",
+                  "expr": "primary_category"
+                },
+                {
+                  "type": "root",
+                  "name": "created_by",
+                  "expr": "created_by"
+                }
+              ]
+            },
+            "dimensionsSpec": {
+              "dimensions": [
+                {
+                  "type": "string",
+                  "name": "program_id"
+                },
+                {
+                  "type": "string",
+                  "name": "status"
+                },
+                {
+                  "type": "string",
+                  "name": "rootorg_id"
+                },
+                {
+                  "type": "string",
+                  "name": "user_id"
+                },
+                {
+                  "type": "string",
+                  "name": "osid"
+                },
+                {
+                  "type": "string",
+                  "name": "user_type"
+                },
+                {
+                  "type": "string",
+                  "name": "contributor_id"
+                },
+                {
+                  "type": "string",
+                  "name": "primary_category"
+                },
+                {
+                  "type": "string",
+                  "name": "created_by"
+                }
+              ],
+              "dimensionsExclusions": []
+            },
+            "timestampSpec": {
+              "column": "timestamp",
+              "format": "auto"
+            }
+          }
+        },
+        "metricsSpec": [
+          {
+            "name": "total_count",
+            "type": "count"
+          }
+        ],
+        "granularitySpec": {
+          "type": "uniform",
+          "segmentGranularity": "day",
+          "queryGranularity": "none",
+          "rollup": true
+        }
+      },
+      "ioConfig": {
+        "type": "index",
+        "firehose": {
+          "type": "static-azure-blobstore",
+          "blobs": [
+            {
+              "container": "reports",
+              "path": "/sourcing/SourcingSummaryReport.json"
+            }
+          ],
+          "fetchTimeout": 300000
+        }
+      },
+      "tuningConfig": {
+        "type": "index",
+        "targetPartitionSize": 5000000,
+        "maxRowsInMemory": 25000,
+        "forceExtendableShardSpecs": false,
+        "logParseExceptions": true
+      }
+    }
+}
diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml
index 35f28ffe4d..df495a5d4a 100644
--- a/ansible/roles/data-products-deploy/tasks/main.yml
+++ b/ansible/roles/data-products-deploy/tasks/main.yml
@@ -1,25 +1,59 @@
 ## Data products deployment ##
+- name: Ensure azure blob storage container exists
+  command: az storage container create --name {{ bucket }}
+  when: dp_object_store_type == "azure"
+  tags:
+    - always
+
 - name: Copy Core Data Products
   copy: src={{ analytics_batch_module_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
     - dataproducts
 
+- name: Copy Core Data Products to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - dataproducts-spark-cluster 
+
 - name: Unarchive Ed Data Products
   become: yes
   unarchive: src={{ playbook_dir}}/{{ analytics_ed_dataporducts_artifact }} dest={{ analytics.home }}/models-{{ model_version }} copy=yes group={{ analytics_group }} owner={{ analytics_user }}
   tags:
-    - ed-dataproducts    
+    - ed-dataproducts
+
+- name: Copy Ed Data Products to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar
+  async: 3600
+  poll: 10
+  tags:
+    - ed-dataproducts-spark-cluster      
 
 - name: Copy Framework Library
   copy: src={{ analytics_core_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
     - framework
 
+- name: Copy Framework Library to azure blob
+  command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster
+
 - name: Copy Scruid Library
   copy: src={{ scruid_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
     - framework  
 
+- name: Copy Scruid Library to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }}
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster  
+
 - name: Copy Job Manager
   copy: src={{ analytics_job_manager_artifact }} dest={{ analytics.home }}/models-{{ model_version }}
   tags:
@@ -31,6 +65,43 @@
     - dataproducts
     - ed-dataproducts
     - framework
+  when: dockdataproducts is undefined
+
+- name: Copy configuration file
+  template: src=common.conf.j2 dest={{ analytics.home }}/models-{{ model_version }}/dock-{{ env }}.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - dataproducts
+    - ed-dataproducts
+    - framework
+  when: dockdataproducts is defined
+
+- name: Copy configuration file as application.conf for cluster
+  template: src=common.conf.j2 dest={{ analytics.home }}/models-{{ model_version }}/application.conf mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - framework-spark-cluster
+
+- name: Update spark temp dir value for cluster
+  lineinfile:
+    path: '{{ analytics.home }}/models-{{ model_version }}/application.conf'
+    regexp: '^spark_output_temp_dir="/mount/data/analytics/tmp/"'
+    line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"'
+  tags:
+    - framework-spark-cluster
+
+- name: Update logger kafka config for cluster
+  lineinfile:
+    path: '{{ analytics.home }}/models-{{ model_version }}/application.conf'
+    regexp: '^log.appender.kafka.enable="false"'
+    line: 'log.appender.kafka.enable="true"'
+  tags:
+    - framework-spark-cluster       
+
+- name: Copy configuration file to azure blob
+  command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf
+  async: 3600
+  poll: 10
+  tags:
+    - framework-spark-cluster    
 
 - name: Copy log4j2 xml file
   template: src=log4j2.xml.j2 dest={{ analytics.home }}/models-{{ model_version }}/log4j2.xml mode=755 owner={{ analytics_user }} group={{ analytics_group }}
@@ -40,6 +111,18 @@
   template: src={{ item }}.j2 dest={{ analytics.home }}/scripts/{{ item }}.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   with_items: "{{ analytics.scripts }}"
   tags: [ dataproducts, framework, ed-dataproducts ]
+  when: dockdataproducts is undefined
+
+- name: Copy python sanity check script file
+  template: src=exhaust_sanity_check.py.j2 dest={{ analytics.home }}/scripts/exhaust_sanity_check.py
+  tags: [ dataproducts, framework, ed-dataproducts ]
+  when: dockdataproducts is undefined
+
+- name: Copy Dock Scripts
+  template: src={{ item }}.j2 dest={{ analytics.home }}/scripts/{{ item }}.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  with_items: "{{ analytics.dockScripts }}"
+  tags: [ dataproducts, framework, ed-dataproducts ]
+  when: dockdataproducts is defined
 
 - name: Update model config
   template: src=model-config.j2 dest={{ analytics.home }}/scripts/model-config.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}
@@ -47,6 +130,15 @@
     - dataproducts
     - update-config
     - ed-dataproducts
+  when: dockdataproducts is undefined
+
+- name: Update model dock config
+  template: src=model-dock-config.j2 dest={{ analytics.home }}/scripts/model-dock-config.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - dataproducts
+    - update-config
+    - ed-dataproducts
+  when: dockdataproducts is defined
 
 - name: Copy submit-all-jobs ruby file
   template: src=submit-all-jobs.rb.j2 dest={{ analytics.home }}/scripts/submit-all-jobs.rb mode=755 owner={{ analytics_user }} group={{ analytics_group }}
@@ -99,29 +191,13 @@
     - spark-jobs
     - cronjobs
 
-- name: Create video-streaming cron job
-  cron: name="{{env}}-video-streaming" minute=*/{{ video_stream_job_schedule }} job="{{ analytics.home }}/scripts/run-job.sh video-streaming"
-  tags:
-    - default-jobs
-    - spark-jobs
-    - cronjobs
-
-- name: Create course-dashboard-metrics cron job
-  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh course-dashboard-metrics"
-  with_dict: "{{ run_course_metrics_job }}"
+- name: Create course-batch-status-updater cron job
+  cron: name="{{env}}-course-batch-status-updater" minute=*/{{ course_batch_status_updater_job_schedule }} job="{{ analytics.home }}/scripts/run-job.sh course-batch-status-updater"
   tags:
     - cronjobs
     - default-jobs
     - spark1-jobs
 
-- name: Create assessment-dashboard-metrics cron job
-  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh assessment-dashboard-metrics"
-  with_dict: "{{ run_assessment_metrics_job }}"
-  tags:
-    - cronjobs
-    - default-jobs
-    - spark-jobs
-
 - name: Create admin-user-reports cron job
   cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh admin-user-reports"
   with_dict: "{{ run_admin_user_reports_job }}"
@@ -129,7 +205,6 @@
     - cronjobs 
     - default-jobs
     - spark-jobs
-    
 - name: Create admin-geo-reports cron job
   cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh admin-geo-reports"
   with_dict: "{{ run_admin_geo_reports_job }}"
@@ -154,24 +229,213 @@
     - default-jobs
     - spark-jobs
 
-#- name: Create district-weekly cron job
-#  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} weekday={{ item.value.weekday }}  job="{{ analytics.home }}/scripts/run-job.sh district-weekly"
-#  with_dict: "{{ run_district_weekly_job }}"
-#  tags:
-#    - cronjobs
-#    - default-jobs
-#    - ed-dataproducts
-
-#- name: Create district-monthly cron job
-#  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} month={{ item.value.month }} day={{ item.value.day }}  job="{{ analytics.home }}/scripts/run-job.sh district-monthly"
-#  with_dict: "{{ run_district_monthly_job }}"
-#  tags:
-#    - cronjobs
-#    - default-jobs
-#    - ed-dataproducts
+- name: Create progress-exhaust cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh progress-exhaust"
+  with_dict: "{{ run_progress_exhaust }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark1-jobs
+
+- name: Create response-exhaust cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh response-exhaust"
+  with_dict: "{{ run_response_exhaust }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+
+- name: Create cassandra-migration cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh cassandra-migration"
+  with_dict: "{{ run_cassandra_migration }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+    
+
+- name: Create userinfo-exhaust cron job
+  cron: name="{{ env }}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }} job="{{ analytics.home }}/scripts/run-job.sh userinfo-exhaust"
+  with_dict: "{{ run_userinfo_exhaust }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+
+- name: Create collection-summary cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh collection-summary-report"
+  with_dict: "{{ run_collection_summary }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+
+- name: Copy collection-summary ingestion spec
+  copy: src="collection-summary-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - ed-dataproducts
+
+- name: Create sourcing-summary cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-dock-job.sh sourcing-summary-report"
+  with_dict: "{{ run_sourcing_summary }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+
+- name: Create uci-private-exhaust cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh uci-private-exhaust"
+  with_dict: "{{ run_uci_private_exhaust_job }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs
+
+- name: Create uci-response-exhaust cron job
+  cron: name="{{env}}-{{ item.key }}" minute={{ item.value.minute }} hour={{ item.value.hour }}  job="{{ analytics.home }}/scripts/run-job.sh uci-response-exhaust"
+  with_dict: "{{ run_uci_response_exhaust_job }}"
+  tags:
+    - cronjobs
+    - default-jobs
+    - spark-jobs 
+
+- name: Copy sourcing-summary ingestion spec
+  copy: src="sourcing-ingestion-spec.json" dest={{ analytics.home }}/scripts/ mode=755 owner={{ analytics_user }} group={{ analytics_group }}
+  tags:
+    - ed-dataproducts
 
 - name: Update start jobmanager
   template: src=start-jobmanager.j2 dest={{ analytics.home }}/scripts/start-jobmanager.sh mode=755 owner={{ analytics_user }} group={{ analytics_group }}
   tags:
     - update-jobmanager-config
-    - dataproducts
\ No newline at end of file
+    - dataproducts
+
+# Cluster job sumbit tasks
+- name: Copy cluster-config.json file
+  template: src=cluster-config.json.j2 dest={{ analytics_cluster.home }}/cluster-config.json
+  delegate_to: localhost
+  tags:
+    - replay-job
+    - run-job
+    - config-update
+
+- name: Copy submit-script.sh file
+  template: src=submit-script.j2 dest={{ analytics_cluster.home }}/submit-script.sh mode=755
+  delegate_to: localhost
+  tags:
+    - replay-job
+    - run-job
+    - config-update
+
+- name: Copy model-config.sh file
+  template: src=model-config.j2 dest={{ analytics_cluster.home }}/model-config.sh
+  delegate_to: localhost  
+  tags:
+    - replay-job
+    - run-job
+    - config-update
+
+- name: Replay Job
+  shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --startDate {{ start_date }} --endDate {{ end_date }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} &"
+  async: "{{ (pause_min * 60) }}"
+  poll: 0
+  tags:
+    - replay-job 
+
+- name: Run Job
+  shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ job_id }} --mode {{ mode }} --partitions {{ partitions }} --parallelisation {{ parallelisation }} --sparkMaster {{ sparkMaster }} --selectedPartitions {{ selected_partitions }} --batch_id {{ batch_id }} &"
+  async: "{{ (pause_min * 60) }}"
+  poll: 0
+  tags:
+    - run-job
+
+- name: Submit jobs
+  shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ item }} --mode default --sparkMaster yarn &"
+  with_items: "{{ jobs.split(',')|list }}"
+  tags:
+    - job-submit
+
+# Cluster exhaust parallel jobs sumbit tasks
+
+- name: Install required python packages
+  pip:
+    name:
+      - psycopg2-binary
+      - pandas
+      - IPython
+  tags:
+    - parallel-jobs-submit  
+
+- name: Copy python script file
+  template: src=update-job-requests.py.j2 dest={{ analytics_cluster.home }}/update-job-requests.py
+  delegate_to: localhost  
+  tags:
+    - parallel-jobs-submit
+
+- name: Execute python script to populate batch numbers
+  shell: |
+    if echo "{{jobs}}" | grep 'druid'
+    then
+     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} druid {{env}}_report_config
+    elif echo "{{jobs}}" | grep 'exhaust'
+     then
+     python {{ analytics_cluster.home }}/update-job-requests.py {{ jobs }} {{ batch_size }} exhaust {{env}}_job_request
+    fi
+  tags:
+    - parallel-jobs-submit
+  register: jobsCountStr
+
+
+- debug:
+    var: jobsCountStr
+  tags:
+    - parallel-jobs-submit  
+
+- name: Get stdout with parallelisation value from python script to tmp file
+  shell: echo "{{ jobsCountStr.stdout }}" > /tmp/test.txt
+  tags:
+    - parallel-jobs-submit
+
+- name: Extract parallelisation value from tmp file
+  shell: "cat /tmp/test.txt | tr '\n' ' ' | awk -F': ' '{print $NF}'"
+  register: jobsCountOut
+  tags:
+    - parallel-jobs-submit
+
+- debug:
+    var: jobsCountOut
+  tags:
+    - parallel-jobs-submit
+
+# set jobs count variable from python script output
+- set_fact:
+    jobs_count: "{{ jobsCountOut.stdout }}"
+  tags:
+    - parallel-jobs-submit  
+
+- name: Submit parallel exhaust jobs
+  shell: "nohup {{ analytics_cluster.home }}/submit-script.sh --job {{ jobs }} --mode parallel-jobs --parallelisation {{ jobs_count }} &"
+  poll: 30
+  tags:
+    - parallel-jobs-submit 
+  register: submitOutput
+
+- debug:
+    var: submitOutput 
+  tags:
+    - parallel-jobs-submit      
+
+# Execute Exhaust job sanity check script tasks
+
+- name: Install required python packages
+  pip:
+    name:
+      - requests
+  tags:
+    - run-sanity    
+
+- name: Run sanity check python script
+  shell: python {{ analytics.home }}/scripts/exhaust_sanity_check.py
+  tags:
+    - run-sanity
+  register: SanityCheckStatus
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2
new file mode 100644
index 0000000000..1a26514684
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2
@@ -0,0 +1,27 @@
+{
+    "jars": [
+        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_core_artifact }}",
+        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ scruid_artifact }}",
+        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_ed_dataporducts_jar_artifact }}"
+    ],
+    "file": "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/{{ analytics_batch_module_artifact }}",
+    "files": [
+        "wasbs://{{ bucket }}@{{sunbird_private_storage_account_name}}.blob.core.windows.net/models-{{ model_version }}/application.conf"
+    ],
+    "className": "org.ekstep.analytics.job.JobExecutor",
+    "executorCores": {{ spark_cluster.executor_core }},
+    "executorMemory": "{{ spark_cluster.executor_memory }}",
+    "numExecutors": {{ spark_cluster.num_executors }},
+    "conf": {
+        "spark.sql.autoBroadcastJoinThreshold" : "-1",
+        "spark.dynamicAllocation.enabled" :"{{ spark_enable_dynamic_allocation }}",
+        "spark.shuffle.service.enabled" :"{{ spark_enable_dynamic_allocation }}",
+        "spark.sql.shuffle.partitions" : "{{ spark_sql_shuffle_partitions }}",
+        "spark.scheduler.mode" : "FAIR",
+        "spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}",
+        "spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}",
+        "spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}",
+        "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}",
+        "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}"
+    }
+}
diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2
index fe271b3e2f..bde88ec9d4 100644
--- a/ansible/roles/data-products-deploy/templates/common.conf.j2
+++ b/ansible/roles/data-products-deploy/templates/common.conf.j2
@@ -9,28 +9,27 @@ spark.cassandra.connection.host="{{groups['dp-cassandra'][0]}}"
 cassandra.keyspace_prefix="{{ cassandra_keyspace_prefix }}"
 cassandra.hierarchy_store_prefix="{{ cassandra_hierarchy_store_prefix }}"
 
-# Content to vec configurations
 
-pc_files_cache="s3"
-pc_dispatch_params="""{"bucket":"{{dp_dispatch_bucket_name}}"}"""
-pc_files_prefix="metrics/"
-
-metrics.dispatch.to="s3"
-metrics.dispatch.params="""{"bucket":"{{dp_dispatch_bucket_name}}"}"""
-metrics.consumption.dataset.id="eks-consumption-metrics/"
-metrics.creation.dataset.id="eks-creation-metrics/"
+storage.key.config="{{ dp_storage_key_config }}"
+storage.secret.config="{{ dp_storage_secret_config }}"
+reports.storage.key.config="{{ dp_reports_storage_key_config }}"
+reports.storage.secret.config="{{ dp_reports_storage_secret_config }}"
+{% if dp_object_store_type == "azure" %}
+cloud_storage_type="azure"
+{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %}
+cloud_storage_type="s3"
+cloud_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}"
+cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}"
+aws_storage_key="{{ s3_storage_key }}"
+aws_storage_secret="{{ s3_storage_secret }}"
+{% endif %}
 
 lp.contentmodel.versionkey="jd5ECm/o0BXwQCe8PfZY1NoUkB9HN41QjA80p22MKyRIcP5RW4qHw8sZztCzv87M"
 
-# Neo4j
-neo4j.bolt.url=""
-neo4j.bolt.user=""
-neo4j.bolt.password=""
-
-# Log4j Kafka appender config
-log4j.appender.kafka.enable="false"
-log4j.appender.kafka.broker_host="{{groups['processing-cluster-kafka'][0]}}:9092"
-log4j.appender.kafka.topic="{{ env }}.telemetry.backend"
+# Joblog Kafka appender config for cluster execution
+log.appender.kafka.enable="false"
+log.appender.kafka.broker_host="{{groups['processing-cluster-kafka'][0]}}:9092"
+log.appender.kafka.topic="{{ env }}.druid.events.log"
 
 # Kafka connection configuration
 kafka.consumer.brokerlist="{{groups['processing-cluster-kafka'][0]}}:9092"
@@ -44,20 +43,6 @@ spark.memory_fraction={{ spark.memory_fraction }}
 spark.storage_fraction={{ spark.storage_fraction }}
 spark.driver_memory="{{ spark.driver_memory }}" 
 
-#Data Exhaust
-
-data_exhaust {
-	save_config {
-		save_type="azure"
-		bucket="{{data_exhaust_blob_name}}"
-		prefix="data-exhaust/"
-		public_s3_url="{{data_exhaust_s3_url}}"
-		local_path="/mount/data/analytics/{{ env }}/tmp/"
-	}
-	delete_source: "true"
-	package.enable: "true"
-}
-
 #Monitor Jobs
 
 monitor {
@@ -75,14 +60,6 @@ default.consumption.app.id="no_value"
 default.channel.id="in.ekstep"
 default.creation.app.id="no_value"
 
-metrics.creation.es.url="http://{{groups['telemetry-search-cluster-kibana-node'][0]}}:9200"
-metrics.creation.es.indexes="compositesearch"
-
-storage-service.request-signature-version="AWS4-HMAC-SHA256"
-s3service.region="ap-south-1"
-
-cloud_storage_type="azure"
-
 
 # Media Service Type
 media_service_type = "azure"
@@ -140,48 +117,39 @@ cloud.container.reports="reports"
 # course metrics container in azure
 course.metrics.cassandra.sunbirdKeyspace="sunbird"
 course.metrics.cassandra.sunbirdCoursesKeyspace="sunbird_courses"
+course.metrics.cassandra.sunbirdHierarchyStore="{{ cassandra_hierarchy_store_keyspace }}"
 course.metrics.cloud.objectKey=""
-course.metrics.temp.dir="/mount/data/analytics/course-reports"
 course.metrics.cassandra.input.consistency="QUORUM"
 es.host="http://{{groups['core-es'][0]}}"
 es.port="9200"
-course.metrics.es.alias="cbatchstats"
-course.metrics.es.index.cbatchstats.prefix="cbatchstats-"
-course.metrics.es.index.cbatch="course-batch"
 es.composite.host="{{groups['composite-search-cluster'][0]}}"
 
 # State admin user reports
 # Uses azure only - course.metrics.cloud.provider
-admin.reports.cloud.container="reports"
 admin.metrics.cloud.objectKey=""
 admin.metrics.temp.dir="/mount/data/analytics/admin-user-reports"
 
 #Assessment report config
-assessment.metrics.temp.dir="/mount/data/analytics/assessment-report"
-assessment.metrics.cassandra.input.consistency="QUORUM"
-assessment.metrics.cloud.objectKey="assessment-reports/"
-assessment.metrics.content.index="compositesearch"
 es.scroll.size = 1000
-assessment.metrics.es.alias="cbatch-assessment"
-assessment.metrics.es.index.prefix="cbatch-assessment-"
-course.upload.reports.enabled=true
-course.es.index.enabled=true
-assessment.metrics.bestscore.report=true // BestScore or Latst Updated Score
+
+#BestScore or Latst Updated Score
+assessment.metrics.bestscore.report=true
 assessment.metrics.supported.contenttype="SelfAssess"
+assessment.metrics.supported.primaryCategories="{{ assessment_metric_primary_category }}"
 spark.sql.caseSensitive=true
 
 # content rating configurations
 
 druid.sql.host="http://{{druid_broker_host}}:8082/druid/v2/sql/"
 druid.unique.content.query="{\"query\":\"SELECT DISTINCT \\\"object_id\\\" AS \\\"Id\\\"\\nFROM \\\"druid\\\".\\\"summary-events\\\" WHERE \\\"__time\\\"  BETWEEN TIMESTAMP '%s' AND TIMESTAMP '%s'\"}"
-druid.content.rating.query="{\"query\":\"SELECT \\\"object_id\\\" AS contentId, COUNT(*) AS \\\"totalRatingsCount\\\", SUM(edata_rating) AS \\\"Total Ratings\\\", SUM(edata_rating)/COUNT(*) AS \\\"averageRating\\\" FROM \\\"druid\\\".\\\"telemetry-feedback-events\\\" WHERE \\\"eid\\\" = 'FEEDBACK' GROUP BY \\\"object_id\\\"\"}"
-druid.content.consumption.query="{\"query\":\"SELECT COUNT(*) as \\\"play_sessions_count\\\", object_id as \\\"contentId\\\", SUM(total_time_spent) as \\\"total_time_spent\\\", dimensions_pdata_id, object_id\\nFROM \\\"summary-events\\\"\\nWHERE \\\"dimensions_mode\\\" = 'play' AND \\\"dimensions_type\\\" ='content'\\nGROUP BY object_id, dimensions_pdata_id\"}"
+druid.content.rating.query="{\"query\":\"SELECT \\\"object_id\\\" AS contentId, COUNT(*) AS \\\"totalRatingsCount\\\", SUM(edata_rating) AS \\\"Total Ratings\\\", SUM(edata_rating)/COUNT(*) AS \\\"averageRating\\\" FROM \\\"druid\\\".\\\"telemetry-feedback-events\\\" WHERE \\\"eid\\\" = 'FEEDBACK' AND \\\"edata_rating\\\">0 GROUP BY \\\"object_id\\\"\"}"
+druid.content.consumption.query="{\"query\":\"SELECT COUNT(*) as \\\"play_sessions_count\\\", object_id as \\\"contentId\\\", SUM(total_time_spent) as \\\"total_time_spent\\\", dimensions_pdata_id, object_id\\nFROM \\\"summary-events\\\"\\nWHERE \\\"dimensions_mode\\\" = 'play' AND \\\"dimensions_type\\\" ='content' AND \\\"dimensions_pdata_pid\\\" != 'creation-portal' \\nGROUP BY object_id, dimensions_pdata_id\"}"
 lp.system.update.base.url="{{lp_url}}/system/v3/content/update"
 
 
 #Experiment Configuration
 
-user.search.api.url="http://{{sunbird_learner_service_url}}/private/user/v1/search"
+user.search.api.url="{{sunbird_learner_service_url}}/private/user/v1/search"
 user.search.limit="10000"
 
 # pipeline auditing
@@ -209,32 +177,132 @@ druid = {
 	url = "/druid/v2/"
 	datasource = "telemetry-events"
 	response-parsing-timeout = 300000
+        client-backend = "com.ing.wbaa.druid.client.DruidAdvancedHttpClient"
+        client-config = {
+           druid-advanced-http-client ={
+           queue-size = 32768
+           queue-overflow-strategy = "Backpressure"
+           query-retries = 5
+           query-retry-delay = 10 ms
+           host-connection-pool = {
+             max-connections = 32
+             min-connections = 0
+             max-open-requests = 128
+             max-connection-lifetime = 20 min
+             idle-timeout = 15 min
+          client = {
+            # The time after which an idle connection will be automatically closed.
+            # Set to `infinite` to completely disable idle timeouts.
+            idle-timeout = 10 min
+           parsing.max-chunk-size = 10m
+        }
+        }
+    }
+
+  }
 }
 druid.rollup.host="{{druid_rollup_broker_host}}"
 druid.rollup.port=8082
-druid.query.wait.time.mins=1
-druid.report.upload.wait.time.mins=1
+druid.query.wait.time.mins=10
+druid.report.upload.wait.time.mins=10
+druid.scan.batch.size=100
+druid.scan.batch.bytes=2000000
+druid.query.batch.buffer=500000
+
 
 // Metric event config
 metric.producer.id="pipeline.monitoring"
 metric.producer.pid="dataproduct.metrics"
 push.metrics.kafka=true
 metric.kafka.broker="{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
-metric.kafka.topic="{{ env }}.telemetry.metrics"
+metric.kafka.topic="{{ env }}.prom.monitoring.metrics"
 
 //Postgres Config
 postgres.db="{{postgres.db_name}}"
 postgres.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/"
 postgres.user="{{postgres.db_username}}"
 postgres.pass="{{postgres.db_password}}"
+postgres.program.table="program"
+postgres.nomination.table="nomination"
+postgres.usertable="\"V_User\""
+postgres.org.table="\"V_User_Org\""
+
+druid.ingestion.path="/druid/indexer/v1/task"
+druid.segment.path="/druid/coordinator/v1/metadata/datasources/"
+druid.deletesegment.path="/druid/coordinator/v1/datasources/"
+
+postgres.druid.db="{{ druid_report_postgres_db_name }}"
+postgres.druid.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/"
+postgres.druid.user="{{ druid_report_postgres_db_username }}"
+postgres.druid.pass="{{ dp_vault_druid_postgress_pass }}"
 
 
 location.search.url="https://{{location_search_url}}/v1/location/search"
 location.search.token="{{ location_search_token }}"
+location.search.request="{\"request\": {\"filters\": {\"type\" :[\"state\",\"district\"]},\"limit\" : 10000}}"
 
 druid.state.lookup.url = "http://{{groups['raw-coordinator'][0]}}:8081/druid/coordinator/v1/lookups/config/__default/stateSlugLookup"
 
+sunbird_encryption_key="{{ core_vault_sunbird_encryption_key }}"
+
 dcedialcode.filename="DCE_dialcode_data.csv"
 etbdialcode.filename="ETB_dialcode_data.csv"
 dcetextbook.filename="DCE_textbook_data.csv"
 etbtextbook.filename="ETB_textbook_data.csv"
+etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }}
+
+
+druid.report.default.storage="azure"
+druid.report.date.format="yyyy-MM-dd"
+druid.report.default.container="report-verification"
+
+## Collection Exhaust Jobs Configuration -- Start ##
+
+sunbird.user.keyspace="{{ user_table_keyspace }}"
+sunbird.courses.keyspace="{{ course_keyspace }}"
+sunbird.content.hierarchy.keyspace="{{ cassandra_hierarchy_store_keyspace }}"
+sunbird.user.cluster.host="{{ core_cassandra_host }}"
+sunbird.courses.cluster.host="{{ core_cassandra_host }}"
+sunbird.content.cluster.host="{{ core_cassandra_host }}"
+sunbird.report.cluster.host="{{ report_cassandra_cluster_host }}"
+sunbird.user.report.keyspace="{{ report_user_table_keyspace }}"
+collection.exhaust.store.prefix=""
+postgres.table.job_request="{{ job_request_table }}"
+postgres.table.dataset_metadata="{{ dataset_metadata_table }}"
+
+## Collection Exhaust Jobs Configuration -- End ##
+
+## Exhaust throttling variables
+exhaust.batches.limit.per.channel={{ exhaust_batches_limit_per_channel }}
+exhaust.file.size.limit.per.channel={{ exhaust_file_size_limit_bytes_per_channel }}
+
+exhaust.parallel.batch.load.limit={{ exhaust_parallel_batch_load_limit }}
+exhaust.user.parallelism={{ exhaust_user_parallelism }}
+
+data_exhaust.batch.limit.per.request={{ data_exhaust_batch_limit_per_request }}
+
+
+
+//START of UCI Postgres Config
+
+uci.conversation.postgres.db="{{ uci_postgres.conversation_db_name }}"
+uci.conversation.postgres.url="jdbc:postgresql://{{uci_postgres.conversation_db_host}}:{{uci_postgres.conversation_db_port}}/"
+
+uci.fushionauth.postgres.db="{{ uci_postgres.fushionauth_db_name }}"
+uci.fushionauth.postgres.url="jdbc:postgresql://{{uci_postgres.fushionauth_db_host}}:{{uci_postgres.fushionauth_db_port}}/"
+
+uci.postgres.table.conversation="{{ uci_postgres.conversation_table_name }}"
+uci.postgres.table.user="{{ uci_postgres.user_table_name }}"
+uci.postgres.table.user_registration="{{ uci_postgres.user_registration_table_name }}"
+uci.postgres.table.identities="{{ uci_postgres.user_identities_table_name }}"
+
+uci.conversation.postgres.user="{{ uci_postgres.conversation_db_user }}"
+uci.conversation.postgres.pass="{{ uci_postgres.conversation_db_psss }}"
+
+uci.fushionauth.postgres.user="{{ uci_postgres.fushionauth_db_user }}"
+uci.fushionauth.postgres.pass="{{ uci_postgres.fushionauth_db_psss }}"
+
+uci.exhaust.store.prefix=""
+uci.encryption.secret="{{ uci_encryption_secret_key }}"
+
+// END OF UCI Related Job Configs
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2 b/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2
new file mode 100644
index 0000000000..3f6ba98d9d
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/exhaust_sanity_check.py.j2
@@ -0,0 +1,58 @@
+import requests
+from requests.auth import HTTPBasicAuth
+import json
+from kafka import KafkaConsumer
+from json import loads
+import sys
+
+def checkClusterStatus():
+	try:
+		res = requests.get('https://{{ spark_cluster_name }}.azurehdinsight.net/api/v1/clusters/{{ spark_cluster_name }}/alerts?format=summary', auth = HTTPBasicAuth("{{ admin_name }}" ,"{{ admin_password }}"))
+		if(res.status_code == 200):
+			resJson = json.loads(res.text)
+			warningCount = resJson["alerts_summary"]["WARNING"]["count"]
+			criticalCount = resJson["alerts_summary"]["CRITICAL"]["count"]
+			unknownCount = resJson["alerts_summary"]["UNKNOWN"]["count"]
+			if((warningCount + criticalCount + unknownCount) == 0):
+				print("Cluster is up & running fine. With these - WARNING:{0}, CRITICAL:{1}, UNKNOWN:{2}".format(warningCount, criticalCount, unknownCount))
+				return "SUCCESS"
+			else:
+				return "FAILED. Cluster is not running properly. Found these - WARNING:{0}, CRITICAL:{1}, UNKNOWN:{2}".format(warningCount, criticalCount, unknownCount)
+		else:
+			return "FAILED. Cluster failed to provide response. Resulted in {0} response".format(res.status_code)    
+	except Exception as e:
+	    return "FAILED with {0}".format(str(e))
+
+def checkCassandraMigratorStatus():
+	try:
+		## from joblog file
+		migratorENDEvent = ""
+		with open ('{{ analytics.home }}/scripts/logs/joblog.log', 'rt') as logs:
+		    for log in logs:
+		    	if (log.count("JOB_END") == 1 & log.count("{{ cassandra_migrator_job_name }}") == 1):
+		    		migratorENDEvent = log
+		logJson = json.loads(migratorENDEvent)
+		jobStatus = logJson["edata"]["status"]
+		if (jobStatus == "SUCCESS"):
+			print("Cassandra Migrator Completed successfully!")
+			return "SUCCESS"
+		else:
+			return "Cassandra Migrator failed"
+	except Exception as e:
+	    return "FAILED with {0}".format(str(e))
+
+
+def main():
+	finalSuccessMessage="All checks are successful"
+	## check Cassandra Migrator status
+	cassandraMigratorState=checkCassandraMigratorStatus()
+	## check spark cluster status
+	clusterState=checkClusterStatus()
+
+	if(cassandraMigratorState == "SUCCESS" and clusterState == "SUCCESS"):
+		return finalSuccessMessage
+	else:
+		raise Exception("Required checks failed. Job Status: {0} and Cluster status: {1}".format(cassandraMigratorState, clusterState))
+
+result=main()
+print(result)
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/generate-metrics.j2 b/ansible/roles/data-products-deploy/templates/generate-metrics.j2
deleted file mode 100644
index d87a454597..0000000000
--- a/ansible/roles/data-products-deploy/templates/generate-metrics.j2
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/sh
-set_env="{{business_metrics.env}}"
-cd {{ analytics.home }}/scripts/BusinessMetrics
-#python template_usage_by_content.py $set_env >> $log_file_path
-python template_usage_by_items.py $set_env
-echo "template_usage_by_items completed."
-python concept_coverage_across_items.py $set_env
-echo "concept_coverage_across_items completed."
-python concept_coverage_across_content.py $set_env
-echo "concept_coverage_across_content completed."
-python asset_usage_in_content.py $set_env
-echo "asset_usage_in_content completed."
-aws s3 cp metrics/ {{business_metrics.s3_path}} --recursive --region ap-south-1
-echo "pushing metrics to s3 completed."
-rm -rf metrics/
-echo "Metrics generated and pushed to S3 successfully."
diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2
index 717caad340..2bb0a042ea 100644
--- a/ansible/roles/data-products-deploy/templates/model-config.j2
+++ b/ansible/roles/data-products-deploy/templates/model-config.j2
@@ -1,77 +1,104 @@
 #!/usr/bin/env bash
 
 config() {
-	bucket={{ bucket }}
-	datasetRawBucket={{ datasetRawBucket }}
-	dataExhaustBucket={{ dataExhaustBucket }}
-	dataExhaustPrefix={{ dataExhaustPrefix }}
-	consumptionRawPrefix={{ consumptionRawPrefix }}
+	bucket={{ secor_bucket }}
 	brokerList={{ brokerlist }}
 	zookeeper={{ zookeeper }}
+	brokerIngestionList={{ ingestion_kafka_brokers }}
 	job_topic={{ analytics_job_queue_topic }}
 	topic={{ topic }}
 	analyticsMetricsTopic={{ analytics_metrics_topic }}
-	learning_topic={{ learning_topic }}
 	sinkTopic={{ sink_topic }}
 	metricsTopic={{ metrics_topic }}
-	currentDate=$(date "+%Y-%m-%d");
 	analytics_home={{ analytics.home }}	
 	temp_folder={{ job_manager_tmp_dir }}
-	sparkCassandraConnectionHost="{{groups['lp-cassandra'][0]}}"
+	sparkCassandraConnectionHost="{{ lp_cassandra_host  }}"
+	sparkRedisConnectionHost={{ metadata2_redis_host }}
 	sunbirdPlatformCassandraHost="{{ core_cassandra_host }}"
 	sunbirdPlatformElasticsearchHost="{{ sunbird_es_host }}"
 	jobManagerJobsCount="{{ analytics_jobs_count }}"
-	dataExhaustChannel="{{ data_exhaust_Channel }}"
-	pipelineAuditSlackUser="Pipeline Audit"
-	druidBrokerUrl="http://{{druid_broker_host}}:8082/"
 	producerEnv="{{ producer_env }}"
 	baseScriptPath="{{ spark_output_temp_dir }}"
 	reportPostContainer="{{ reports_container }}"
-	
+	druidIngestionURL="{{ druid_rollup_cluster_ingestion_task_url }}/druid/indexer/v1/task"
+    assessTopic={{ assess_topic }}
+
+
 	if [ -z "$2" ]; then endDate=$(date --date yesterday "+%Y-%m-%d"); else endDate=$2; fi
 	if [ ! -z "$3" ]; then inputBucket=$3; fi
 	if [ ! -z "$4" ]; then sinkTopic=$4; fi
+	if [ ! -z "$2" ]; then keyword=$2; fi
 	case "$1" in
-		"cmu")
-       	if [ -z "$2" ]; then queryDate=$currentDate; else queryDate=$endDate; fi
-       	echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.updater.ContentModelUpdater","modelParams":{"date":"'$queryDate'","contentTypes":{"Content":["Resource", "Collection", "TextBook", "LessonPlan", "Course", "Template", "Asset", "Plugin", "LessonPlanUnit","CourseUnit", "TextBookUnit"]}},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$learning_topic'"}}],"parallelization":10,"appName":"Content Model Updater","deviceMapping":false}'
-        ;;
-		"data-exhaust")
-		#echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.model.DataExhaustJobModel","parallelization":8,"appName":"Data Exhaust","deviceMapping":false,"modelParams":{"shouldDelay":true,"delayInMilis":1800000},"exhaustConfig":{"eks-consumption-raw":{"events":["eks-consumption-raw"],"eventConfig":{"eks-consumption-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"tags":{"name":"tags","operator":"IN"},"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-consumption-summary":{"events":["ME_SESSION_SUMMARY","ME_CONTENT_USAGE_SUMMARY","ME_GENIE_LAUNCH_SUMMARY","ME_ITEM_SUMMARY","ME_GENIE_USAGE_SUMMARY","ME_ITEM_USAGE_SUMMARY","ME_ASSET_SNAPSHOT_SUMMARY","ME_CONTENT_SNAPSHOT_SUMMARY","ME_DEVICE_USAGE_SUMMARY","ME_GENIE_SESSION_SUMMARY"],"eventConfig":{"ME_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ss/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.timeSpent":{"to":"Time spent on the content"},"edata.eks.screenSummary":{"hidden":true},"edata.eks.currentLevel":{"to":"Current Domain Level"},"edata.eks.syncDate":{"mapFunc":"timestampToDateTime","to":"Sync Date(IST)"},"edata.eks.timeDiff":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Content Start Time(IST)"},"uid":{"to":"Genie User ID"},"ets":{"hidden":true},"dimensions.group_user":{"to":"Is Group User"},"edata.eks.activitySummary":{"to":"Activity Summary"},"edata.eks.end_time":{"hidden":true},"edata.eks.eventsSummary":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.levels":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.itemResponses":{"to":"edata.eks.itemResponses"},"dimensions.gdata.id":{"to":"Content ID"},"edata.eks.noOfLevelTransitions":{"to":"Number of Level Transitions"},"edata.eks.noOfAttempts":{"to":"Number of Attempts"},"tags":{"to":"Genie Tags"},"edata.eks.interruptTime":{"to":"Total interrupt time"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time stamp(IST)"},"edata.eks.mimeType":{"hidden":true},"edata.eks.telemetryVersion":{"to":"Telemetry Version"},"context.pdata.ver":{"hidden":true},"dimensions.gdata.ver":{"hidden":true},"edata.eks.interactEventsPerMin":{"to":"Number of interactions per minute"},"edata.eks.contentType":{"to":"Content Type"},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in User"},"dimensions.did":{"to":"Device ID"},"dimensions.loc":{"to":"Lat / Long"},"mid":{"to":"Session ID"},"edata.eks.noOfInteractEvents":{"to":"Number of Interact events in the content session"},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_LAUNCH_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"gls/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ITEM_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"is/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"genie-launch-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"to":"UIDs"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.total_ts":{"to":"Total time (secs) per tag for the period"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.contents":{"to":"Update col header\tContent IDs of all contents used per tag for the period"},"context.pdata.id":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Average time spent per Genie session"},"edata.eks.device_ids":{"to":"Device IDs per tag for the period"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.tag":{"to":"Tag name"},"edata.eks.total_sessions":{"to":"Total number of sessions per tag for the period"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true},"dimensions.period":{"to":"period"}}}},"ME_ITEM_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"item-usage-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ASSET_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"asset-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.used_activities_count":{"to":"No. of activities that are part of lessons"},"edata.eks.used_templates_count":{"to":"No. of templates that are part of lessons"},"context.date_range.from":{"hidden":true},"uid":{"to":"uid"},"edata.eks.total_questions_count":{"to":"Total number of questions"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.used_audio_count":{"to":"No. of audio files that are being used"},"edata.eks.used_images_count":{"to":"No. of images that are being used"},"edata.eks.used_questions_count":{"to":"No. of questions that are being used"},"edata.eks.total_activities_count":{"to":"Total number of activities"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_templates_count":{"to":"Total no. of templates"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"edata.eks.total_images_count":{"to":"Total no. of images"},"context.pdata.model":{"hidden":true},"edata.eks.total_audio_count":{"to":"Total no. of audio files"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"content-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.active_user_count":{"to":"No. of active authors on the portal"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.author_id":{"to":"Author ID"},"edata.eks.total_user_count":{"to":"Total number of registered authors on the portal"},"edata.eks.live_content_count":{"to":"Total number of content pieces live"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_content_count":{"to":"Total number of content pieces"},"edata.eks.review_content_count":{"to":"Total number of content pieces pending review"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_DEVICE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"dus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.total_play_time":{"to":"Total Play time on device"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp of first Genie installation on device(IST)"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.avg_num_launches":{"to":"Average num. of Genie launches per day"},"edata.eks.num_contents":{"to":"No. of content pieces available on device"},"edata.eks.play_start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device first started game play(IST)"},"edata.eks.end_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used(IST)"},"eid":{"hidden":true},"edata.eks.num_days":{"to":"Number of days since Genie installed before the timestamp indicating the last use"},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.mean_play_time":{"to":"Average play time on device"},"edata.eks.mean_play_time_interval":{"to":"Average time interval between game play on the device"},"edata.eks.last_played_on":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used for game play(IST)"},"edata.eks.avg_time":{"to":"Average time on Genie per day"},"edata.eks.last_played_content":{"to":"Last played content on device"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.did":{"to":"Device ID"},"edata.eks.num_sessions":{"to":"Total number of sessions from the device"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_GENIE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.content":{"to":"Content IDs of content used in Genie Session"},"edata.eks.timeSpent":{"to":"Time spent on Genie Session\t"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.group_user":{"to":"Is Group user?"},"edata.eks.contentCount":{"to":"Number of content pieces used"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.time_stamp":{"mapFunc":"timestampToDateTime","to":"Session End time stamp(IST)"},"context.pdata.id":{"hidden":true},"tags":{"to":"Genie tags"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in user?"},"dimensions.did":{"to":"Device ID"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true}}}}}},"eks-consumption-metrics":{"events":["ME_CONTENT_USAGE_METRICS","ME_ITEM_USAGE_METRICS","ME_GENIE_USAGE_METRICS","ME_CONTENT_SNAPSHOT_METRICS","ME_CONCEPT_SNAPSHOT_METRICS","ME_ASSET_SNAPSHOT_METRICS"],"eventConfig":{"ME_CONTENT_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"dimensions.tag":{"to":"tag"},"edata.eks.m_publish_date":{"to":"Date when the content is published"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_avg_ts_session":{"to":"Average timespent per session"},"edata.eks.m_total_interactions":{"to":"Total interactions count"},"edata.eks.m_avg_interactions_min":{"to":"Average interactions per minute"},"edata.eks.m_total_devices":{"to":"Total number of devices"},"edata.eks.m_avg_sess_device":{"to":"Average sessions per device"}}}},"ME_ITEM_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_item_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"tag"},"dimensions.content_id":{"to":"Content ID"},"dimensions.item_id":{"to":"item id"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_count":{"to":"Total response count"},"edata.eks.m_correct_res_count":{"to":"Correct response count"},"edata.eks.m_inc_res_count":{"to":"Incorrect response count"},"edata.eks.m_correct_res":{"hidden":true},"edata.eks.m_incorrect_res":{"hidden":true},"edata.eks.m_top5_incorrect_res":{"to":"Top 5 Incorrect response"},"edata.eks.m_avg_ts":{"to":"Average time spent"},"edata.eks.m_top5_misconception_res":{"to":"Top 5 Misconception response"}}}},"ME_GENIE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_genie_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"Device Tag"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_avg_ts_session":{"to":"Average sessions per device"},"edata.eks.m_contents":{"hidden":true},"edata.eks.m_total_devices":{"to":"Total number of devices"}}}},"ME_CONTENT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_CONCEPT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_concept_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_ASSET_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_asset_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}}}},"eks-creation-raw":{"events":["eks-creation-raw"],"eventConfig":{"eks-creation-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-creation-summary":{"events":["ME_APP_SESSION_SUMMARY","ME_CE_SESSION_SUMMARY","ME_TEXTBOOK_SESSION_SUMMARY","ME_APP_USAGE_SUMMARY","ME_CE_USAGE_SUMMARY","ME_TEXTBOOK_USAGE_SUMMARY","ME_AUTHOR_USAGE_SUMMARY","ME_PUBLISH_PIPELINE_SUMMARY"],"eventConfig":{"ME_APP_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"app-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.interact_events_per_min":{"to":"Average ineractions per minute"},"edata.eks.events_summary":{"to":"edata.eks.events_summary"},"dimensions.app_id":{"to":"Event Source"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Start Time(IST)"},"uid":{"to":"User ID"},"edata.eks.page_summary":{"to":"Pages visited Summary"},"ets":{"hidden":true},"edata.eks.page_views_count":{"to":"Total pageviews in Session"},"edata.eks.end_time":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.ce_visits":{"to":"Number of Authoring tool visits"},"dimensions.sid":{"to":"Session ID"},"edata.eks.time_diff":{"to":"Session length "},"edata.eks.interact_events_count":{"to":"Number of Interactions"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.time_spent":{"to":"Session length"},"dimensions.anonymous_user":{"to":"User Logged In"},"mid":{"to":"mid"},"edata.eks.env_summary":{"to":"Usage summary"},"edata.eks.first_visit":{"to":"First Time Visitor"},"context.date_range.to":{"hidden":true}}}},"ME_CE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ce-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"textbook-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_APP_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"app-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"edata.eks.avg_ts_session":{"to":"Average session duration per day"},"context.granularity":{"to":"Granularity"},"edata.eks.anon_total_sessions":{"to":"Time spent by non-registered users"},"edata.eks.unique_users":{"to":"Unique users list"},"dimensions.app_id":{"to":"Source"},"edata.eks.unique_users_count":{"to":"Unique users count"},"context.date_range.from":{"hidden":true},"edata.eks.total_pageviews_count":{"to":"Total Page Views per day"},"uid":{"to":"uid"},"edata.eks.ce_total_sessions":{"to":"Number of sessions where creation happened"},"edata.eks.anon_avg_ts_session":{"to":"Avg. session duration for non-registered users per day"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.total_ts":{"to":"Total time spent by visitors on Portal"},"edata.eks.ce_percent_sessions":{"to":"Percentage of visitors creating content"},"dimensions.author_id":{"to":"Author ID"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.new_user_count":{"to":"Number of new users"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"edata.eks.percent_new_users_count":{"to":"Percentage of new visitors per day"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_pageviews":{"to":"Avg. pages per visit per day"},"edata.eks.total_sessions":{"to":"Total number of sessions"},"edata.eks.anon_total_ts":{"to":"Total time spent by non-registered users on Portal"},"mid":{"hidden":true},"dimensions.period":{"to":"dimensions.period"},"context.date_range.to":{"hidden":true}}}},"ME_CE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"ce-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"textbook-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_AUTHOR_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"author-usage-summary/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"uid":{"to":"Author ID"},"mid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.id":{"hidden":true},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Total number of seesion per day"},"edata.eks.total_ts":{"to":"Total time spent by author"},"edata.eks.ce_total_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_sessions":{"to":"No of sessions per Author"},"context.granularity":{"hidden":true},"edata.eks.ce_percent_ts":{"to":"Percentage of time spent browsing vs creating"},"edata.eks.ce_percent_sessions":{"to":"Percentage of sessions in which users log into content editor"},"edata.eks.ce_total_visits":{"to":"No of Content Editor visits per author"},"context.date_range.from":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_PUBLISH_PIPELINE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"publish-pipeline-summ/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.publish_pipeline_summary":{"to":"edata.eks.publish_pipeline_summary"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"dimensions.period":{"to":"Date"},"context.date_range.to":{"hidden":true}}}}}},"eks-creation-metrics":{"events":["ME_APP_USAGE_METRICS","ME_CE_USAGE_METRICS","ME_TEXTBOOK_CREATION_METRICS","ME_TEXTBOOK_SNAPSHOT_METRICS","ME_AUTHOR_USAGE_METRICS"],"eventConfig":{"ME_APP_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_app_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID / All"},"dimensions.app_id":{"to":"App (EkStep portal currently)"},"edata.eks.anon_total_sessions":{"to":"Total anonymous sessions on portal"},"edata.eks.anon_total_ts":{"to":"Tota time spent (Anonymous) on portal"},"edata.eks.total_sessions":{"to":"Total number of portal sessions"},"edata.eks.total_ts":{"to":"Total time spent on portal"},"edata.eks.ce_total_sessions":{"to":"Total content editor sessions"},"edata.eks.ce_percent_sessions":{"to":"% of total sessions that are content editor sessions"},"edata.eks.total_pageviews_count":{"to":"Total pageviews on portal"},"edata.eks.unique_users":{"hidden":true},"edata.eks.unique_users_count":{"to":"Count of unique users on portal"},"edata.eks.avg_pageviews":{"to":"Avg. number of pageviews per session for specified period"},"edata.eks.avg_ts_session":{"to":"Avg. session length (time spent) per session for specified period"},"edata.eks.anon_avg_ts_session":{"to":"Avg. Anonymous session length (time spent) per session for specified period"},"edata.eks.new_user_count":{"to":"No. of new users on the app (portal) for the given time period"},"edata.eks.percent_new_users_count":{"to":"% of new users for the given time period"}}}},"ME_CE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_ce_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"edata.eks.unique_users_count":{"to":"Number of users who accessed the content piece via the editor"},"edata.eks.total_sessions":{"to":"Total number of sessions that accessed the content piece via the editor"},"edata.eks.total_ts":{"to":"Time spent on the content piece in the content editor"},"edata.eks.avg_ts_session":{"to":"Average session time on the editor"},"edata.eks.update_date":{"to":"Last updated timestamp of the content piece"}}}},"ME_TEXTBOOK_CREATION_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_creation_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_TEXTBOOK_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_AUTHOR_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_author_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":" Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID /All"},"edata.eks.total_sessions":{"to":"Total number of portal sessions by author"},"edata.eks.total_ts":{"to":"Total time spent on portal by author"},"edata.eks.total_ce_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_ce_visit":{"to":"Number of content editor visits"},"edata.eks.percent_ce_sessions":{"to":"% of sessions that were spent on the editor"},"edata.eks.avg_ts_session":{"to":"Average time spent by author on portal per session"},"edata.eks.percent_ce_ts":{"to":"% time spent by author on content editor"}}}}}}}}'
-		echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.model.DataExhaustJobModel","parallelization":8,"appName":"Data Exhaust","deviceMapping":false,"modelParams":{"shouldDelay":true,"delayInMilis":1800000},"exhaustConfig":{"eks-consumption-raw":{"events":["eks-consumption-raw"],"eventConfig":{"eks-consumption-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"channel/"}},"filterMapping":{"tags":{"name":"tags","operator":"IN"},"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-consumption-summary":{"events":["ME_SESSION_SUMMARY","ME_CONTENT_USAGE_SUMMARY","ME_GENIE_LAUNCH_SUMMARY","ME_ITEM_SUMMARY","ME_GENIE_USAGE_SUMMARY","ME_ITEM_USAGE_SUMMARY","ME_ASSET_SNAPSHOT_SUMMARY","ME_CONTENT_SNAPSHOT_SUMMARY","ME_DEVICE_USAGE_SUMMARY","ME_GENIE_SESSION_SUMMARY","ME_WORKFLOW_SUMMARY"],"eventConfig":{"ME_WORKFLOW_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/wfs/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"dimensions.channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ss/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.timeSpent":{"to":"Time spent on the content"},"edata.eks.screenSummary":{"hidden":true},"edata.eks.currentLevel":{"to":"Current Domain Level"},"edata.eks.syncDate":{"mapFunc":"timestampToDateTime","to":"Sync Date(IST)"},"edata.eks.timeDiff":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Content Start Time(IST)"},"uid":{"to":"Genie User ID"},"ets":{"hidden":true},"dimensions.group_user":{"to":"Is Group User"},"edata.eks.activitySummary":{"to":"Activity Summary"},"edata.eks.end_time":{"hidden":true},"edata.eks.eventsSummary":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.levels":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.itemResponses":{"to":"edata.eks.itemResponses"},"dimensions.gdata.id":{"to":"Content ID"},"edata.eks.noOfLevelTransitions":{"to":"Number of Level Transitions"},"edata.eks.noOfAttempts":{"to":"Number of Attempts"},"tags":{"to":"Genie Tags"},"edata.eks.interruptTime":{"to":"Total interrupt time"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time stamp(IST)"},"edata.eks.mimeType":{"hidden":true},"edata.eks.telemetryVersion":{"to":"Telemetry Version"},"context.pdata.ver":{"hidden":true},"dimensions.gdata.ver":{"hidden":true},"edata.eks.interactEventsPerMin":{"to":"Number of interactions per minute"},"edata.eks.contentType":{"to":"Content Type"},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in User"},"dimensions.did":{"to":"Device ID"},"dimensions.loc":{"to":"Lat / Long"},"mid":{"to":"Session ID"},"edata.eks.noOfInteractEvents":{"to":"Number of Interact events in the content session"},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_LAUNCH_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/gls/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ITEM_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/is/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_GENIE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/genie-launch-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"to":"UIDs"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.total_ts":{"to":"Total time (secs) per tag for the period"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.contents":{"to":"Update col header\tContent IDs of all contents used per tag for the period"},"context.pdata.id":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Average time spent per Genie session"},"edata.eks.device_ids":{"to":"Device IDs per tag for the period"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.tag":{"to":"Tag name"},"edata.eks.total_sessions":{"to":"Total number of sessions per tag for the period"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true},"dimensions.period":{"to":"period"}}}},"ME_ITEM_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/item-usage-summ/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_ASSET_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/asset-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.used_activities_count":{"to":"No. of activities that are part of lessons"},"edata.eks.used_templates_count":{"to":"No. of templates that are part of lessons"},"context.date_range.from":{"hidden":true},"uid":{"to":"uid"},"edata.eks.total_questions_count":{"to":"Total number of questions"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.used_audio_count":{"to":"No. of audio files that are being used"},"edata.eks.used_images_count":{"to":"No. of images that are being used"},"edata.eks.used_questions_count":{"to":"No. of questions that are being used"},"edata.eks.total_activities_count":{"to":"Total number of activities"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_templates_count":{"to":"Total no. of templates"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"edata.eks.total_images_count":{"to":"Total no. of images"},"context.pdata.model":{"hidden":true},"edata.eks.total_audio_count":{"to":"Total no. of audio files"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_CONTENT_SNAPSHOT_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/content-snapshot/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.active_user_count":{"to":"No. of active authors on the portal"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.author_id":{"to":"Author ID"},"edata.eks.total_user_count":{"to":"Total number of registered authors on the portal"},"edata.eks.live_content_count":{"to":"Total number of content pieces live"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"edata.eks.total_content_count":{"to":"Total number of content pieces"},"edata.eks.review_content_count":{"to":"Total number of content pieces pending review"},"dimensions.partner_id":{"to":"Partner ID"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_DEVICE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/dus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.total_play_time":{"to":"Total Play time on device"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp of first Genie installation on device(IST)"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"edata.eks.avg_num_launches":{"to":"Average num. of Genie launches per day"},"edata.eks.num_contents":{"to":"No. of content pieces available on device"},"edata.eks.play_start_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device first started game play(IST)"},"edata.eks.end_time":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used(IST)"},"eid":{"hidden":true},"edata.eks.num_days":{"to":"Number of days since Genie installed before the timestamp indicating the last use"},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.mean_play_time":{"to":"Average play time on device"},"edata.eks.mean_play_time_interval":{"to":"Average time interval between game play on the device"},"edata.eks.last_played_on":{"mapFunc":"timestampToDateTime","to":"Time stamp when the device was last used for game play(IST)"},"edata.eks.avg_time":{"to":"Average time on Genie per day"},"edata.eks.last_played_content":{"to":"Last played content on device"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.did":{"to":"Device ID"},"edata.eks.num_sessions":{"to":"Total number of sessions from the device"},"mid":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_GENIE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/cus/"}},"filterMapping":{"tags":{"name":"genieTag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"edata.eks.content":{"to":"Content IDs of content used in Genie Session"},"edata.eks.timeSpent":{"to":"Time spent on Genie Session\t"},"uid":{"to":"uid"},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"dimensions.group_user":{"to":"Is Group user?"},"edata.eks.contentCount":{"to":"Number of content pieces used"},"eid":{"hidden":true},"ver":{"hidden":true},"edata.eks.time_stamp":{"mapFunc":"timestampToDateTime","to":"Session End time stamp(IST)"},"context.pdata.id":{"hidden":true},"tags":{"to":"Genie tags"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"dimensions.anonymous_user":{"to":"Logged in user?"},"dimensions.did":{"to":"Device ID"},"mid":{"to":"Unique Genie Session ID"},"context.date_range.to":{"hidden":true}}}}}},"eks-consumption-metrics":{"events":["ME_CONTENT_USAGE_METRICS","ME_ITEM_USAGE_METRICS","ME_GENIE_USAGE_METRICS","ME_CONTENT_SNAPSHOT_METRICS","ME_CONCEPT_SNAPSHOT_METRICS","ME_ASSET_SNAPSHOT_METRICS"],"eventConfig":{"ME_CONTENT_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"dimensions.tag":{"to":"tag"},"edata.eks.m_publish_date":{"to":"Date when the content is published"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_avg_ts_session":{"to":"Average timespent per session"},"edata.eks.m_total_interactions":{"to":"Total interactions count"},"edata.eks.m_avg_interactions_min":{"to":"Average interactions per minute"},"edata.eks.m_total_devices":{"to":"Total number of devices"},"edata.eks.m_avg_sess_device":{"to":"Average sessions per device"}}}},"ME_ITEM_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_item_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"tag"},"dimensions.content_id":{"to":"Content ID"},"dimensions.item_id":{"to":"item id"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_total_count":{"to":"Total response count"},"edata.eks.m_correct_res_count":{"to":"Correct response count"},"edata.eks.m_inc_res_count":{"to":"Incorrect response count"},"edata.eks.m_correct_res":{"hidden":true},"edata.eks.m_incorrect_res":{"hidden":true},"edata.eks.m_top5_incorrect_res":{"to":"Top 5 Incorrect response"},"edata.eks.m_avg_ts":{"to":"Average time spent"},"edata.eks.m_top5_misconception_res":{"to":"Top 5 Misconception response"}}}},"ME_GENIE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_genie_usage_metrics/"}},"filterMapping":{"tags":{"name":"dimensions.tag","operator":"IN"},"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.tag":{"to":"Device Tag"},"edata.eks.m_total_sessions":{"to":"Total number of sessions"},"edata.eks.m_total_ts":{"to":"Total time spent"},"edata.eks.m_avg_ts_session":{"to":"Average sessions per device"},"edata.eks.m_contents":{"hidden":true},"edata.eks.m_total_devices":{"to":"Total number of devices"}}}},"ME_CONTENT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_content_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_CONCEPT_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_concept_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_ASSET_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-consumption-metrics/me_asset_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}}}},"eks-creation-raw":{"events":["eks-creation-raw"],"eventConfig":{"eks-creation-raw":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"unique/"}},"filterMapping":{"channel":{"name":"context.channel","operator":"EQ"},"app_id":{"name":"context.pdata.id","operator":"EQ"}}}}},"eks-creation-summary":{"events":["ME_APP_SESSION_SUMMARY","ME_CE_SESSION_SUMMARY","ME_TEXTBOOK_SESSION_SUMMARY","ME_APP_USAGE_SUMMARY","ME_CE_USAGE_SUMMARY","ME_TEXTBOOK_USAGE_SUMMARY","ME_AUTHOR_USAGE_SUMMARY","ME_PUBLISH_PIPELINE_SUMMARY"],"eventConfig":{"ME_APP_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/app-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"edata.eks.interact_events_per_min":{"to":"Average ineractions per minute"},"edata.eks.events_summary":{"to":"edata.eks.events_summary"},"dimensions.app_id":{"to":"Event Source"},"context.date_range.from":{"hidden":true},"edata.eks.start_time":{"mapFunc":"timestampToDateTime","to":"Start Time(IST)"},"uid":{"to":"User ID"},"edata.eks.page_summary":{"to":"Pages visited Summary"},"ets":{"hidden":true},"edata.eks.page_views_count":{"to":"Total pageviews in Session"},"edata.eks.end_time":{"hidden":true},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.ce_visits":{"to":"Number of Authoring tool visits"},"dimensions.sid":{"to":"Session ID"},"edata.eks.time_diff":{"to":"Session length "},"edata.eks.interact_events_count":{"to":"Number of Interactions"},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.time_spent":{"to":"Session length"},"dimensions.anonymous_user":{"to":"User Logged In"},"mid":{"to":"mid"},"edata.eks.env_summary":{"to":"Usage summary"},"edata.eks.first_visit":{"to":"First Time Visitor"},"context.date_range.to":{"hidden":true}}}},"ME_CE_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ce-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_SESSION_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/textbook-ss/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_APP_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/app-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"edata.eks.avg_ts_session":{"to":"Average session duration per day"},"context.granularity":{"to":"Granularity"},"edata.eks.anon_total_sessions":{"to":"Time spent by non-registered users"},"edata.eks.unique_users":{"to":"Unique users list"},"dimensions.app_id":{"to":"Source"},"edata.eks.unique_users_count":{"to":"Unique users count"},"context.date_range.from":{"hidden":true},"edata.eks.total_pageviews_count":{"to":"Total Page Views per day"},"uid":{"to":"uid"},"edata.eks.ce_total_sessions":{"to":"Number of sessions where creation happened"},"edata.eks.anon_avg_ts_session":{"to":"Avg. session duration for non-registered users per day"},"ets":{"mapFunc":"timestampToDateTime","to":"Generation Time(IST)"},"edata.eks.total_ts":{"to":"Total time spent by visitors on Portal"},"edata.eks.ce_percent_sessions":{"to":"Percentage of visitors creating content"},"dimensions.author_id":{"to":"Author ID"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.new_user_count":{"to":"Number of new users"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"edata.eks.percent_new_users_count":{"to":"Percentage of new visitors per day"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_pageviews":{"to":"Avg. pages per visit per day"},"edata.eks.total_sessions":{"to":"Total number of sessions"},"edata.eks.anon_total_ts":{"to":"Total time spent by non-registered users on Portal"},"mid":{"hidden":true},"dimensions.period":{"to":"dimensions.period"},"context.date_range.to":{"hidden":true}}}},"ME_CE_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/ce-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_TEXTBOOK_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/textbook-usage/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":false,"columnMappings":{}}},"ME_AUTHOR_USAGE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/author-usage-summary/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"uid":{"to":"Author ID"},"mid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"syncts":{"mapFunc":"timestampToDateTime","to":"Sync time(IST)"},"context.pdata.id":{"hidden":true},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"edata.eks.avg_ts_session":{"to":"Total number of seesion per day"},"edata.eks.total_ts":{"to":"Total time spent by author"},"edata.eks.ce_total_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_sessions":{"to":"No of sessions per Author"},"context.granularity":{"hidden":true},"edata.eks.ce_percent_ts":{"to":"Percentage of time spent browsing vs creating"},"edata.eks.ce_percent_sessions":{"to":"Percentage of sessions in which users log into content editor"},"edata.eks.ce_total_visits":{"to":"No of Content Editor visits per author"},"context.date_range.from":{"hidden":true},"context.date_range.to":{"hidden":true}}}},"ME_PUBLISH_PIPELINE_SUMMARY":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"derived/publish-pipeline-summ/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"context.granularity":{"hidden":true},"context.date_range.from":{"hidden":true},"uid":{"hidden":true},"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"eid":{"hidden":true},"ver":{"hidden":true},"context.pdata.id":{"hidden":true},"edata.eks.publish_pipeline_summary":{"to":"edata.eks.publish_pipeline_summary"},"syncts":{"to":"Event sync time(IST)","mapFunc":"timestampToDateTime"},"context.pdata.ver":{"hidden":true},"context.pdata.model":{"hidden":true},"mid":{"hidden":true},"dimensions.period":{"to":"Date"},"context.date_range.to":{"hidden":true}}}}}},"eks-creation-metrics":{"events":["ME_APP_USAGE_METRICS","ME_CE_USAGE_METRICS","ME_TEXTBOOK_CREATION_METRICS","ME_TEXTBOOK_SNAPSHOT_METRICS","ME_AUTHOR_USAGE_METRICS"],"eventConfig":{"ME_APP_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_app_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID / All"},"dimensions.app_id":{"to":"App (EkStep portal currently)"},"edata.eks.anon_total_sessions":{"to":"Total anonymous sessions on portal"},"edata.eks.anon_total_ts":{"to":"Tota time spent (Anonymous) on portal"},"edata.eks.total_sessions":{"to":"Total number of portal sessions"},"edata.eks.total_ts":{"to":"Total time spent on portal"},"edata.eks.ce_total_sessions":{"to":"Total content editor sessions"},"edata.eks.ce_percent_sessions":{"to":"% of total sessions that are content editor sessions"},"edata.eks.total_pageviews_count":{"to":"Total pageviews on portal"},"edata.eks.unique_users":{"hidden":true},"edata.eks.unique_users_count":{"to":"Count of unique users on portal"},"edata.eks.avg_pageviews":{"to":"Avg. number of pageviews per session for specified period"},"edata.eks.avg_ts_session":{"to":"Avg. session length (time spent) per session for specified period"},"edata.eks.anon_avg_ts_session":{"to":"Avg. Anonymous session length (time spent) per session for specified period"},"edata.eks.new_user_count":{"to":"No. of new users on the app (portal) for the given time period"},"edata.eks.percent_new_users_count":{"to":"% of new users for the given time period"}}}},"ME_CE_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_ce_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":"Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.content_id":{"to":"Content ID"},"edata.eks.unique_users_count":{"to":"Number of users who accessed the content piece via the editor"},"edata.eks.total_sessions":{"to":"Total number of sessions that accessed the content piece via the editor"},"edata.eks.total_ts":{"to":"Time spent on the content piece in the content editor"},"edata.eks.avg_ts_session":{"to":"Average session time on the editor"},"edata.eks.update_date":{"to":"Last updated timestamp of the content piece"}}}},"ME_TEXTBOOK_CREATION_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_creation_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_TEXTBOOK_SNAPSHOT_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_textbook_snapshot_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}}},"ME_AUTHOR_USAGE_METRICS":{"searchType":"azure","fetchConfig":{"params":{"bucket":"'$bucket'","prefix":"eks-creation-metrics/me_author_usage_metrics/"}},"filterMapping":{"channel":{"name":"channel","operator":"EQ"},"app_id":{"name":"dimensions.pdata.id","operator":"EQ"}},"csvConfig":{"auto_extract_column_names":true,"columnMappings":{"ets":{"mapFunc":"timestampToDateTime","to":"Event generation Time(IST)"},"syncts":{"mapFunc":"timestampToDateTime","to":"Event sync time(IST)"},"eid":{"hidden":true},"dimensions.period":{"to":" Time period of the summary. For ex: Day or Week or Month represented by a number"},"dimensions.author_id":{"to":"Author ID /All"},"edata.eks.total_sessions":{"to":"Total number of portal sessions by author"},"edata.eks.total_ts":{"to":"Total time spent on portal by author"},"edata.eks.total_ce_ts":{"to":"Total time spent on content editor by author"},"edata.eks.total_ce_visit":{"to":"Number of content editor visits"},"edata.eks.percent_ce_sessions":{"to":"% of sessions that were spent on the editor"},"edata.eks.avg_ts_session":{"to":"Average time spent by author on portal per session"},"edata.eks.percent_ce_ts":{"to":"% time spent by author on content editor"}}}}}}}}'
+        "assessment-correction")
+         echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}'
+         ;;
+	    "assessment-archival")
+	    echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"azure","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}'
+	    ;;
+	    "assessment-archived-removal")
+	    echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"azure","blobExt":"csv.gz","reportPath":"archived-data/","container":"reports"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}'
+	    ;;
+        "collection-reconciliation-job")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CollectionReconciliationJob","modelParams":{"mode":"prodrun","brokerList":"{{ingestion_kafka_broker_host}}","topic":"{{env}}.issue.certificate.request","sparkCassandraConnectionHost":"{{ core_cassandra_host }}"},"parallelization":30,"appName":"CollectionReconciliationJob"}'
+		;;
+		"collection-summary-report")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"], "contentType": "Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
+		;;
+		"score-metric-migration-job")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}'
+		;;
+		"assessment-score-metric-correction")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}'
+		;;
+		"course-batch-status-updater")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"azure","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}'
+		;;
+		"collection-summary-report-v2")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"druid_storage_account_key","storageSecretConfig":"druid_storage_account_secret","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"azure","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}'
+		;;
+		"uci-private-exhaust")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","storageContainer":"reports","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}'
+		;;
+		"uci-response-exhaust")
+		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"azure","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":"","storageContainer":"reports"},"parallelization":8,"appName":"UCI Response Exhaust"}'
+		;;
+        "userinfo-exhaust")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}'
+		;;
+		"program-collection-summary-report")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"], "keywords":"'$keyword'", "store":"azure","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}'
+		;;
+		"response-exhaust")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}'
+		;;
+    "response-exhaust-v2")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}'
+		;;
+		"progress-exhaust")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}'
+		;;
+		"progress-exhaust-v2")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"azure","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}'
+		;;
+		"druid_reports")
+		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"mode":"batch"},"parallelization":8,"appName":"Druid Reports"}'
+		;;
+		"cassandra-migration")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.updater.CassandraMigratorJob","modelParams":{"cassandraDataHost":"{{ core_cassandra_host }}","cassandraMigrateHost":"{{ report_cassandra_host }}","keyspace":"sunbird_courses","cassandraDataTable":"user_enrolments","cassandraMigrateTable":"{{ report_user_enrolment_table }}","repartitionColumns":"batchid"},"parallelization":10,"appName":"Cassandra Migrator","deviceMapping":false}'
 		;;
         "monitor-job-summ")        
-        #echo '{"search":{"type":"local","queries":[{"file":"/mnt/data/analytics/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"model":[{"model":"ItemSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"GenieUsageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"GenieStageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"ItemUsageSummaryModel","category":"consumption","input_dependency":"ItemSummaryModel"},{"model":"DeviceContentUsageSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"DeviceUsageSummaryModel","category":"consumption","input_dependency":"GenieLaunchSummaryModel"},{"model":"UpdateGenieUsageDB","category":"consumption","input_dependency":"GenieUsageSummaryModel"},{"model":"UpdateItemSummaryDB","category":"consumption","input_dependency":"ItemUsageSummaryModel"},{"model":"UpdateContentUsageDB","category":"consumption","input_dependency":"ContentUsageSummaryModel"},{"model":"UpdateContentPopularityDB","category":"consumption","input_dependency":"ContentPopularitySummaryModel"},{"model":"ContentUsageSummaryModel","category":"consumption","input_dependency":"LearnerSessionSummaryModel"},{"model":"UpdateLearnerProfileDB","category":"consumption","input_dependency":"None"},{"model":"UpdateDeviceSpecificationDB","category":"consumption","input_dependency":"None"},{"model":"LearnerSessionSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieSessionSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieLaunchSummaryModel","category":"consumption","input_dependency":"None"},{"model":"ContentPopularitySummaryModel","category":"consumption","input_dependency":"None"},{"model":"EOCRecommendationFunnelModel","category":"consumption","input_dependency":"None"},{"model":"StageSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieFunnelAggregatorModel","category":"consumption","input_dependency":"None"},{"model":"ContentSideloadingSummaryModel","category":"consumption","input_dependency":"None"},{"model":"GenieFunnelModel","category":"consumption","input_dependency":"None"},{"model":"UpdateContentModel","category":"consumption","input_dependency":"None"},{"model":"ConsumptionMetricsUpdater","category":"consumption","input_dependency":"None"},{"model":"PrecomputedViews","category":"consumption","input_dependency":"None"},{"model":"DataExhaustJob","category":"consumption","input_dependency":"None"},{"model":"AppUsageSummaryModel","category":"creation","input_dependency":"AppSessionSummaryModel"},{"model":"ContentEditorUsageSummaryModel","category":"creation","input_dependency":"ContentEditorSessionSummaryModel"},{"model":"TextbookUsageSummaryModel","category":"creation","input_dependency":"TextbookSessionSummaryModel"},{"model":"UpdateAppUsageDB","category":"creation","input_dependency":"AppUsageSummaryModel"},{"model":"UpdateContentEditorUsageDB","category":"creation","input_dependency":"ContentEditorUsageSummaryModel"},{"model":"UpdateTextbookUsageDB","category":"creation","input_dependency":"TextbookUsageSummaryModel"},{"model":"UpdateAuthorSummaryDB","category":"creation","input_dependency":"AuthorUsageSummaryModel"},{"model":"UpdatePublishPipelineSummarycreation","category":"creation","input_dependency":"PublishPipelineSummaryModel"},{"model":"AppSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"ContentEditorSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"PublishPipelineSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateObjectLifecycleDB","category":"creation","input_dependency":"None"},{"model":"UpdateContentCreationMetricsDB","category":"creation","input_dependency":"None"},{"model":"AuthorUsageSummaryModel","category":"creation","input_dependency":"None"},{"model":"TextbookSessionSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateCreationMetricsDB","category":"creation","input_dependency":"None"},{"model":"UpdateConceptSnapshotDB","category":"creation","input_dependency":"None"},{"model":"ContentSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"ConceptSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateContentSnapshotDB","category":"creation","input_dependency":"None"},{"model":"AssetSnapshotSummaryModel","category":"creation","input_dependency":"None"},{"model":"UpdateTextbookSnapshotDB","category":"creation","input_dependency":"None"},{"model":"UpdateAssetSnapshotDB","category":"creation","input_dependency":"None"},{"model":"ContentLanguageRelationModel","category":"creation","input_dependency":"None"},{"model":"ConceptLanguageRelationModel","category":"creation","input_dependency":"None"},{"model":"AuthorRelationsModel","category":"creation","input_dependency":"None"},{"model":"CreationRecommendationEnrichmentModel","category":"creation","input_dependency":"None"},{"model":"ContentAssetRelationModel","category":"creation","input_dependency":"None"},{"model":"DeviceRecommendationTrainingModel","category":"recommendation","input_dependency":"None"},{"model":"DeviceRecommendationScoringModel","category":"recommendation","input_dependency":"None"},{"model":"ContentVectorsModel","category":"recommendation","input_dependency":"None"},{"model":"EndOfContentRecommendationModel","category":"recommendation","input_dependency":"None"},{"model":"CreationRecommendationModel","category":"recommendation","input_dependency":"None"},{"model":"UpdatePluginSnapshotDB","category":"creation","input_dependency":"None"},{"model":"UpdateTemplateSnapshotDB","category":"creation","input_dependency":"None"},{"model":"AppUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"AssetSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"CEUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"ConceptSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ContentSnapshotMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ContentUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"GenieUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"ItemUsageMetricCreationModel","category":"consumption","input_dependency":"None"},{"model":"TextbookSnapshotMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"TextbookUsageMetricCreationModel","category":"creation","input_dependency":"None"},{"model":"AuthorUsageMetricCreationModel","category":"creation","input_dependency":"None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}'
-        echo '{"search":{"type":"local","queries":[{"file":"'$analytics_home'/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"pushMetrics":true,"brokerList":"'$brokerList'","topic":"'$analyticsMetricsTopic'","model":[{"model":"DeviceSummaryModel","category":"consumption","input_dependency":"None"},{"model":"UpdateDeviceProfileDB","category":"consumption","input_dependency":"DeviceSummaryModel"},{"model":"DataExhaustJob","category":"consumption","input_dependency":"None"},{"model":"UpdateWorkFlowUsageDB","category":"consumption","input_dependency":"WorkFlowUsageSummaryModel"},{"model":"WorkFlowSummaryModel","category":"consumption","input_dependency":"None"},{"model":"WorkFlowUsageSummaryModel","category":"consumption","input_dependency":"WorkFlowSummaryModel"},{"model":"UpdatePortalMetrics","category":"consumption","input_dependency":"None"},{"model":"UpdateWorkFlowUsageMetricsModel","category":"consumption","input_dependency":"None"},{"model":"CourseMetricsJob","category":"consumption","input_dependency":"None"},{"model":"UpdateContentRating","category":"consumption","input_dependency":"None"},{"model":"DruidQueryProcessingModel","category":"consumption","input_dependency":"None"},{"model":"DistrictMonthlyJob","category":"consumption","input_dependency":"None"},{"model":"DesktopConsumptionDailyMetricsJob","category":"consumption","input_dependency":"None"},{"model":"DistrictWeeklyJob","category":"consumption","input_dependency":"None"},{"model":"ConsumptionDailyMetricsJob","category":"consumption","input_dependency":"None"},{"model":"MetricsAuditJob","category":"consumption","input_dependency":"None"},{"model":"AssessmentMetricsJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminReportJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminGeoReportJob","category":"consumption","input_dependency":"None"},{"model": "CourseEnrollmentJob","category": "consumption","input_dependency": "None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}'
+        echo '{"search":{"type":"local","queries":[{"file":"'$analytics_home'/scripts/logs/joblog.log"}]},"model":"org.ekstep.analytics.model.MonitorSummaryModel","modelParams":{"pushMetrics":true,"brokerList":"'$brokerList'","topic":"'$analyticsMetricsTopic'","model":[{"model":"WorkFlowSummaryModel","category":"consumption","input_dependency":"None"},{"model":"UpdateContentRating","category":"consumption","input_dependency":"None"},{"model":"DruidQueryProcessingModel","category":"consumption","input_dependency":"None"},{"model":"MetricsAuditJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminReportJob","category":"consumption","input_dependency":"None"},{"model":"StateAdminGeoReportJob","category":"consumption","input_dependency":"None"},{"model":"CourseEnrollmentJob","category":"consumption","input_dependency":"None"}]},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"appName":"TestMonitorSummarizer","deviceMapping":true}'
         ;;
 		"job-manager")
 		echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}'
 		;;
 		"wfs")
-		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2", "parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}'
+		echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"{{ dp_storage_key_config }}", "storageSecretConfig":"{{ dp_storage_secret_config }}", "apiVersion":"v2", "parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"} }],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}'
 		#echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}'
 	   	;;
-	   	"wfus")
-		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowUsageSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Usage Summarizer","deviceMapping":false}'
-	   	;;
-	   	"wfu")
-		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfus/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.updater.UpdateWorkFlowUsageDB","output":[{"to":"console","params":{"printEvent":false}}],"parallelization":10,"appName":"Workflow Usage Updater","deviceMapping":false}'
-		;;
-		"ds")
-		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0},{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.DeviceSummaryModel","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":12,"appName":"Device Summarizer","deviceMapping":false}'
-	   	;;
-	   	"dpu")
-		echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/device-summary/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.UpdateDeviceProfileDB","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}}],"parallelization":8,"appName":"Device Profile Updater","deviceMapping":false}'
-	   	;;
 		"video-streaming")
 		echo '{"search":{"type":"azure"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}'
 	   	;;
-	   	"course-dashboard-metrics")
-		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CourseMetricsJob","modelParams":{"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"channel","aliasName":"channel"}],"filters":[{"type":"equals","dimension":"contentType","value":"Course"},{"type":"isnotnull","dimension":"identifier","value":""},{"type":"isnotnull","dimension":"channel","value":""}],"descending":"false"},"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Course Dashboard Metrics","deviceMapping":false}'
-		;;
-		"assessment-dashboard-metrics")
-		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.AssessmentMetricsJob","modelParams":{"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"channel","aliasName":"channel"}],"filters":[{"type":"equals","dimension":"contentType","value":"Course"},{"type":"isnotnull","dimension":"identifier","value":""},{"type":"isnotnull","dimension":"channel","value":""}],"descending":"false"},"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Assessment Dashboard Metrics","deviceMapping":false}'
-		;;
 		"admin-user-reports")
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}'
         ;;
@@ -85,38 +112,32 @@ config() {
         echo '{"search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}'
         ;;
         "content-rating-updater")
-		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.updater.UpdateContentRating","modelParams":{"startDate":"'$endDate'","endDate":"'$endDate'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Rating Updater","deviceMapping":false}'
+		echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.updater.UpdateContentRating","modelParams": {"startDate": "'$endDate'","endDate": "'$endDate'"},"output": [{"to":"console","params":{"printEvent":false}}],"parallelization": 8,"appName": "Content Rating Updater","deviceMapping": false}'
 		;;
         "experiment")
 		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.ExperimentDefinitionModel","modelParams":{"sparkElasticsearchConnectionHost":"{{ lp_composite_search_host }}"},"output":[{"to":"elasticsearch","params":{"index":"experiment"}}],"parallelization":8,"appName":"Experiment-Definition","deviceMapping":false}'
-        ;;
-        "district-monthly")
-        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"district_level_report","queryType":"groupBy","dateRange":{"staticInterval":"LastMonth","granularity":"all"},"metrics":[{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastMonth","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}}],"labels":{"state":"State","district":"District","total_unique_devices":"Number of Unique Devices"},"output":[{"type":"csv","metrics":["total_unique_devices"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":true}],"mergeConfig":{"frequency":"MONTH","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"district_report.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}'
-        ;;
-        "district-weekly")
-        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"district_weekly_report","queryType":"groupBy","dateRange":{"staticInterval":"LastWeek","granularity":"all"},"metrics":[{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnPortal","label":"Total Unique Devices on Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnApp","label":"Total Unique Devices on App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastWeek","aggregations":[{"name":"total_unique_devices_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScans","label":"Total QR Scans","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"context_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScansOnPortal","label":"Total QR Scans on Portal","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalQRScansOnApp","label":"Total QR Scans on App","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_scans_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"eid","value":"SEARCH"},{"type":"in","dimension":"object_type","values":["qr","Qr","DialCode","dialcode"]},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlays","label":"Total Content Plays","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.portal","'$producerEnv'.app"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlaysOnPortal","label":"Total Content Plays on Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}},{"metric":"totalContentPlaysOnApp","label":"Total Content Plays on App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastWeek","aggregations":[{"name":"total_content_plays_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"derived_loc_state","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"stateSlugLookup","replaceMissingValueWith":"Unknown"}]},{"fieldName":"derived_loc_district","aliasName":"district","type":"cascade","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.toLowerCase().trim().split(\" \").map(function(t){return t.substring(0,1).toUpperCase()+t.substring(1,t.length)}).join(\" \")}"},{"type":"registeredLookup","fn":"districtLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"isnotnull","dimension":"derived_loc_state"},{"type":"isnotnull","dimension":"derived_loc_district"}],"descending":"false"}}],"labels":{"state":"State","district":"District","total_unique_devices":"Total Unique Devices","total_unique_devices_on_portal":"Total Unique Devices on Portal","total_unique_devices_on_app":"Total Unique Devices on App","total_scans":"Total QR Scans","total_scans_on_portal":"Total QR Scans on Portal","total_scans_on_app":"Total QR Scans on App","total_content_plays":"Total Content Plays","total_content_plays_on_portal":"Total Content Plays on Portal","total_content_plays_on_app":"Total Content Plays on App"},"output":[{"type":"csv","metrics":["total_unique_devices_on_app","total_unique_devices_on_portal","total_unique_devices","total_scans_on_app","total_scans_on_portal","total_scans","total_content_plays_on_app","total_content_plays_on_portal","total_content_plays"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":true}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"district_weekly.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}'
         ;;
 		"etb-metrics")
-		echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.model.report.ETBMetricsJob","modelParams": {"reportConfig": {"id": "etb_metrics","metrics": [],"labels": {"date": "Date","identifier": "TextBook ID","name": "TextBook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","totalQRCodes": "Total number of QR codes","contentLinkedQR": "Number of QR codes with atleast 1 linked content","withoutContentQR": "Number of QR codes with no linked content","withoutContentT1": "Term 1 QR Codes with no linked content","withoutContentT2": "Term 2 QR Codes with no linked content","status": "Status","totalContentLinked": "Total content linked","totalQRLinked": "Total QR codes linked to content","totalQRNotLinked": "Total number of QR codes with no linked content","leafNodesCount": "Total number of leaf nodes","leafNodeUnlinked": "Number of leaf nodes with no content","l1Name": "Level 1 Name","l2Name": "Level 2 Name","l3Name": "Level 3 Name","l4Name": "Level 4 Name","l5Name": "Level 5 Name","dialcode": "QR Code","sum(scans)": "Total Scans","noOfContent": "Number of contents","nodeType": "Type of Node","term": "Term"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}],"mergeConfig": {"frequency": "WEEK","basePath": "'$baseScriptPath'","rollup": 0,"reportPath": "dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig": {"id": "etb_metrics","metrics": [],"labels": {},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}],"mergeConfig": {"frequency": "WEEK","basePath": "'$baseScriptPath'","rollup": 1,"reportPath": "dialcode_counts.csv","rollupAge": "ACADEMIC_YEAR","rollupCol": "Date","rollupRange": 10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig": {"bucket": "'$reportPostContainer'","file": "dialcode_scans/dialcode_counts.csv"},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live", "Draft", "Review"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"tenantConfig": {"tenantId": "","slugName": ""},"store": "azure","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$bucket'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "ETB Metrics Model","deviceMapping": false}'
-		;;
-		"daily-metrics")
-        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"ETB-Consumption-Daily-Reports","queryType":"groupBy","dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalPercentFailedScans","label":"Total Percent Failed QR Scans","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-rollup-syncts","intervals":"LastDay","aggregations":[{"type":"filtered","filterAggType":"longSum","fieldName":"total_count","filterFieldName":"edata_size","filterValue":0,"name":"total_failed_scans"},{"type":"filtered","filterAggType":"longSum","fieldName":"total_count","filterFieldName":"edata_size","filterValue":1,"name":"total_successful_scans"},{"name":"total_scans","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"dialcode_channel","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"object_type","values":["DialCode","dialcode","qr","Qr"]},{"type":"equals","dimension":"eid","value":"SEARCH"}],"postAggregation":[{"type":"javascript","name":"total_percent_failed_scans","fields":{"leftField":"total_failed_scans","rightField":"total_scans","rightFieldType":"FieldAccess"},"fn":"function(total_failed_scans, total_scans) { return (total_scans > 0) ? (total_failed_scans/total_scans) * 100 : 0 }"}],"descending":"false"}},{"metric":"totalContentDownload","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-events-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_download","type":"count","fieldName":""}],"dimensions":[{"fieldName":"content_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"edata_subtype","value":"ContentDownload-Success"},{"type":"equals","dimension":"eid","value":"INTERACT"},{"type":"in","dimension":"context_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"object_type","value":"Resource"}],"descending":"false"}},{"metric":"totalContentPlayed","label":"Total Content Played","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedOnApp","label":"Total Content Played On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays_on_app","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedOnPortal","label":"Total Content Played On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"total_content_plays_on_portal","type":"longSum","fieldName":"total_count"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalUniqueDevices","label":"Total Unique Devices","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnApp","label":"Total Unique Devices On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalUniqueDevicesOnPortal","label":"Total Unique Devices On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_unique_devices_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"app"}],"descending":"false"}},{"metric":"totalDevicesPlayingContent","label":"Total Devices Playing Content","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"total_devices_playing_content","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalDevicesPlayingContentOnApp","label":"Total Devices Playing Content On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"devices_playing_content_on_app","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalDevicesPlayingContentOnPortal","label":"Total Devices Playing Content On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","aggregations":[{"name":"devices_playing_content_on_portal","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"descending":"false"}},{"metric":"totalContentPlayedInHour","label":"Total Content Played In Hour","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnApp","label":"Total Content Played In Hour On App","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.app"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours_on_app","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnPortal","label":"Total Content Played In Hour On Portal","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"total_time_spent"}],"dimensions":[{"fieldName":"collection_created_for","aliasName":"state","type":"extraction","extractionFn":[{"type":"registeredlookup","fn":"channelSlugLookup","replaceMissingValueWith":"Unknown"}]}],"filters":[{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.portal"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"}],"postAggregation":[{"type":"javascript","name":"total_time_spent_in_hours_on_portal","fields":{"leftField":"sum__edata_time_spent","rightField":"sum__edata_time_spent","rightFieldType":"FieldAccess"},"fn":"function(delta, time_spent) { return Math.round(delta/3600) }"}],"descending":"false"}}],"labels":{"state":"State","total_content_download":"Total Content Downloads","total_successful_scans":"Number of successful QR Scans","total_percent_failed_scans":"Total Percent Failed Scans","total_unique_devices":"Total Unique Devices","total_unique_devices_on_app":"Total Unique Devices On App","total_unique_devices_on_portal":"Total Unique Devices On Portal","total_time_spent_in_hours":"Total Content Play Time (in hours)","total_time_spent_in_hours_on_app":"Total Content Play Time on App (in hours)","total_time_spent_in_hours_on_portal":"Total Content Play Time on Portal (in hours)","total_failed_scans":"Number of failed QR Scans","total_scans":"Number of QR Scans","total_content_plays":"Total Content Plays","total_content_plays_on_app":"Total Content Plays On App","total_content_plays_on_portal":"Total Content Plays On Portal","total_app_sessions":"Total App Sesions","time_spent_on_app_in_hours":"Total Time on App (in hours)","devices_playing_content_on_app":"Total Devices that Played Content on App","devices_playing_content_on_portal":"Total Devices that Played Content on Portal","total_devices_playing_content":"Total Devices that Played Content"},"output":[{"type":"csv","metrics":["total_scans","total_successful_scans","total_failed_scans","total_percent_failed_scans","total_content_download","total_content_plays_on_app","devices_playing_content_on_app","total_time_spent_in_hours_on_app","total_content_plays_on_portal","devices_playing_content_on_portal","total_time_spent_in_hours_on_portal","total_content_plays","total_devices_playing_content","total_time_spent_in_hours","total_unique_devices_on_app","total_unique_devices_on_portal","total_unique_devices"],"dims":["state","date"],"fileParameters":["id","dims"],"locationMapping":false}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"daily_metrics.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}'
-        ;;
-        "desktop-consumption-report")
-        echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"reportConfig":{"id":"Desktop-Consumption-Daily-Reports","queryType":"groupBy","dateRange":{"staticInterval":"LastDay","granularity":"day"},"metrics":[{"metric":"totalContentDownloadDesktop","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"telemetry-events-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_content_download_on_desktop","type":"count","fieldName":"mid"}],"dimensions":[{"fieldName":"content_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"context_env","value":"downloadManager"},{"type":"equals","dimension":"edata_state","value":"COMPLETED"},{"type":"equals","dimension":"context_pdata_id","value":"'$producerEnv'.desktop"},{"type":"equals","dimension":"eid","value":"AUDIT"}],"descending":"false"}},{"metric":"totalContentPlayedDesktop","label":"Total time spent in hours","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_content_plays_on_desktop","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"descending":"false"}},{"metric":"totalContentPlayedInHourOnDesktop","label":"Total Content Download","druidQuery":{"queryType":"groupBy","dataSource":"summary-rollup-syncts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"postAggregation":[{"type":"arithmetic","name":"total_time_spent_in_hours_on_desktop","fields":{"leftField":"sum__edata_time_spent","rightField":3600,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}},{"metric":"totalUniqueDevicesPlayedContentOnDesktop","label":"Total Unique Devices On Desktop that played content","druidQuery":{"queryType":"groupBy","dataSource":"summary-distinct-counts","intervals":"LastDay","granularity":"all","aggregations":[{"name":"total_unique_devices_on_desktop_played_content","type":"HLLSketchMerge","fieldName":"unique_devices","lgk":12,"tgtHLLType":"HLL_4","round":true}],"dimensions":[{"fieldName":"collection_board","aliasName":"state"}],"filters":[{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_pdata_id","value":"'$producerEnv'.desktop"}],"descending":"false"}}],"labels":{"state":"State","total_content_plays_on_desktop":"Total Content Played","total_content_download_on_desktop":"Total Content Downloads","total_time_spent_in_hours_on_desktop":"Total time spent in hours","total_unique_devices_on_desktop_played_content":"Total Unique Devices On Desktop that played content"},"output":[{"type":"csv","label":"desktop","metrics":["total_content_download_on_desktop","total_time_spent_in_hours_on_desktop","total_content_plays_on_desktop","total_unique_devices_on_desktop_played_content"],"dims":["state","date"],"fileParameters":["dims"],"locationMapping":false}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":1,"rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":1,"reportPath":"desktop_consumption_report.csv","postContainer":"'$reportPostContainer'"}},"store":"azure","container":"'$bucket'","key":"druid-reports/"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Druid Query Processor","deviceMapping":false}'
+		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}'
 		;;
 		"course-enrollment-report")
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}'
          ;;
-		 "course-consumption-report")
+		"course-consumption-report")
         echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams": {"esConfig": {"request": {"filters": {"objectType": ["Content"],"contentType": ["Course"],"identifier": [],"status": ["Live"]}}},"reportConfig": {"id": "tpd_metrics","labels": {"date": "Date","status": "Batch Status","timespent": "Timespent in mins","courseName": "Course Name","batchName": "Batch Name"},"dateRange": {"staticInterval": "LastDay","granularity": "all"},"metrics": [{"metric": "totalCoursePlays","label": "Total Course Plays (in mins)","druidQuery": {"queryType": "groupBy","dataSource": "summary-events","intervals":"LastDay","aggregations": [{"name": "sum__edata_time_spent","type": "doubleSum","fieldName": "edata_time_spent"}],"dimensions": [{"fieldName": "object_rollup_l1","aliasName": "courseId"}, {"fieldName": "uid","aliasName": "userId"}, {"fieldName": "context_cdata_id","aliasName": "batchId"}],"filters": [{"type": "equals","dimension": "eid","value": "ME_WORKFLOW_SUMMARY"}, {"type": "in","dimension": "dimensions_pdata_id","values": ["'$producerEnv'.app", "'$producerEnv'.portal"]}, {"type": "equals","dimension": "dimensions_type","value": "content"}, {"type": "equals","dimension": "dimensions_mode","value": "play"}, {"type": "equals","dimension": "context_cdata_type","value": "batch"}],"postAggregation": [{"type": "arithmetic","name": "timespent","fields": {"leftField": "sum__edata_time_spent","rightField": 60,"rightFieldType": "constant"},"fn": "/"}],"descending": "false"}}],"output": [{"type": "csv","metrics": ["timespent"],"dims": []}],"queryType": "groupBy"},"store": "azure","format":"csv","key": "druid-reports/","filePath": "druid-reports/","container":"'$bucket'","folderPrefix": ["slug", "reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "TPD Course Consumption Metrics Model","deviceMapping": false}'
          ;;
-         "textbook-progress-report")
+        "textbook-progress-report")
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent": "Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"azure","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}'
          ;;
         "audit-metrics-report")
         echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"azure","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"azure","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}'
          ;;
+		"sourcing-metrics")
+		echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","primaryCategory":"Collection Category","identifier": "Collection ID","name": "Collection Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Folder Name","totalChapters": "Total number of first level folders","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "in","dimension": "primaryCategory","values": ["Digital Textbook", "Course", "Content Playlist","Question paper","Question Paper"]}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "azure","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}'
+		 ;;
+	  "druid-dataset")
+		echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"azure","container":"'$reportPostContainer'","key":"ml_reports/","format":"csv"},"output":[{"to": "console","params": {"printEvent": false}}],"parallelization":8,"appName":"ML Druid Data Model"}'
+		;;
         "*")
 		echo "Unknown model code"
       	exit 1 # Command to come out of the program with status 1
diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2
index 09c7312231..4594a1978a 100644
--- a/ansible/roles/data-products-deploy/templates/model-config.json.j2
+++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2
@@ -1,1860 +1,11 @@
 {
-  "data-exhaust": {
-    "search": {
-      "type": "azure"
-    },
-    "model": "org.ekstep.analytics.model.DataExhaustJobModel",
-    "parallelization": 8,
-    "appName": "Data Exhaust",
-    "deviceMapping": false,
-    "modelParams": {
-      "shouldDelay": true,
-      "delayInMilis": 1800000
-    },
-    "exhaustConfig": {
-      "eks-consumption-raw": {
-        "events": [
-          "eks-consumption-raw"
-        ],
-        "eventConfig": {
-          "eks-consumption-raw": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "channel/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "tags",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "context.channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "context.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          }
-        }
-      },
-      "eks-consumption-summary": {
-        "events": [
-          "ME_SESSION_SUMMARY",
-          "ME_CONTENT_USAGE_SUMMARY",
-          "ME_GENIE_LAUNCH_SUMMARY",
-          "ME_ITEM_SUMMARY",
-          "ME_GENIE_USAGE_SUMMARY",
-          "ME_ITEM_USAGE_SUMMARY",
-          "ME_ASSET_SNAPSHOT_SUMMARY",
-          "ME_CONTENT_SNAPSHOT_SUMMARY",
-          "ME_DEVICE_USAGE_SUMMARY",
-          "ME_GENIE_SESSION_SUMMARY",
-          "ME_WORKFLOW_SUMMARY"
-        ],
-        "eventConfig": {
-          "ME_WORKFLOW_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/wfs/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "dimensions.channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          },
-          "ME_SESSION_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/ss/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "edata.eks.timeSpent": {
-                  "to": "Time spent on the content"
-                },
-                "edata.eks.screenSummary": {
-                  "hidden": true
-                },
-                "edata.eks.currentLevel": {
-                  "to": "Current Domain Level"
-                },
-                "edata.eks.syncDate": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync Date(IST)"
-                },
-                "edata.eks.timeDiff": {
-                  "hidden": true
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.start_time": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Content Start Time(IST)"
-                },
-                "uid": {
-                  "to": "Genie User ID"
-                },
-                "ets": {
-                  "hidden": true
-                },
-                "dimensions.group_user": {
-                  "to": "Is Group User"
-                },
-                "edata.eks.activitySummary": {
-                  "to": "Activity Summary"
-                },
-                "edata.eks.end_time": {
-                  "hidden": true
-                },
-                "edata.eks.eventsSummary": {
-                  "hidden": true
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "edata.eks.levels": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.itemResponses": {
-                  "to": "edata.eks.itemResponses"
-                },
-                "dimensions.gdata.id": {
-                  "to": "Content ID"
-                },
-                "edata.eks.noOfLevelTransitions": {
-                  "to": "Number of Level Transitions"
-                },
-                "edata.eks.noOfAttempts": {
-                  "to": "Number of Attempts"
-                },
-                "tags": {
-                  "to": "Genie Tags"
-                },
-                "edata.eks.interruptTime": {
-                  "to": "Total interrupt time"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time stamp(IST)"
-                },
-                "edata.eks.mimeType": {
-                  "hidden": true
-                },
-                "edata.eks.telemetryVersion": {
-                  "to": "Telemetry Version"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "dimensions.gdata.ver": {
-                  "hidden": true
-                },
-                "edata.eks.interactEventsPerMin": {
-                  "to": "Number of interactions per minute"
-                },
-                "edata.eks.contentType": {
-                  "to": "Content Type"
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "dimensions.anonymous_user": {
-                  "to": "Logged in User"
-                },
-                "dimensions.did": {
-                  "to": "Device ID"
-                },
-                "dimensions.loc": {
-                  "to": "Lat / Long"
-                },
-                "mid": {
-                  "to": "Session ID"
-                },
-                "edata.eks.noOfInteractEvents": {
-                  "to": "Number of Interact events in the content session"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_CONTENT_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/cus/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_GENIE_LAUNCH_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/gls/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_ITEM_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/is/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_GENIE_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/genie-launch-summ/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "uid": {
-                  "to": "UIDs"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation time(IST)"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Total time (secs) per tag for the period"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "edata.eks.contents": {
-                  "to": "Update col header\tContent IDs of all contents used per tag for the period"
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.avg_ts_session": {
-                  "to": "Average time spent per Genie session"
-                },
-                "edata.eks.device_ids": {
-                  "to": "Device IDs per tag for the period"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "dimensions.tag": {
-                  "to": "Tag name"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "Total number of sessions per tag for the period"
-                },
-                "mid": {
-                  "to": "Unique Genie Session ID"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "period"
-                }
-              }
-            }
-          },
-          "ME_ITEM_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/item-usage-summ/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_ASSET_SNAPSHOT_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/asset-snapshot/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "edata.eks.used_activities_count": {
-                  "to": "No. of activities that are part of lessons"
-                },
-                "edata.eks.used_templates_count": {
-                  "to": "No. of templates that are part of lessons"
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "uid": {
-                  "to": "uid"
-                },
-                "edata.eks.total_questions_count": {
-                  "to": "Total number of questions"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Generation Time(IST)"
-                },
-                "edata.eks.used_audio_count": {
-                  "to": "No. of audio files that are being used"
-                },
-                "edata.eks.used_images_count": {
-                  "to": "No. of images that are being used"
-                },
-                "edata.eks.used_questions_count": {
-                  "to": "No. of questions that are being used"
-                },
-                "edata.eks.total_activities_count": {
-                  "to": "Total number of activities"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "edata.eks.total_templates_count": {
-                  "to": "Total no. of templates"
-                },
-                "dimensions.partner_id": {
-                  "to": "Partner ID"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "edata.eks.total_images_count": {
-                  "to": "Total no. of images"
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "edata.eks.total_audio_count": {
-                  "to": "Total no. of audio files"
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_CONTENT_SNAPSHOT_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/content-snapshot/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.active_user_count": {
-                  "to": "No. of active authors on the portal"
-                },
-                "uid": {
-                  "to": "uid"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation time(IST)"
-                },
-                "dimensions.author_id": {
-                  "to": "Author ID"
-                },
-                "edata.eks.total_user_count": {
-                  "to": "Total number of registered authors on the portal"
-                },
-                "edata.eks.live_content_count": {
-                  "to": "Total number of content pieces live"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "edata.eks.total_content_count": {
-                  "to": "Total number of content pieces"
-                },
-                "edata.eks.review_content_count": {
-                  "to": "Total number of content pieces pending review"
-                },
-                "dimensions.partner_id": {
-                  "to": "Partner ID"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_DEVICE_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/dus/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "edata.eks.total_play_time": {
-                  "to": "Total Play time on device"
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.start_time": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Time stamp of first Genie installation on device(IST)"
-                },
-                "uid": {
-                  "to": "uid"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation time(IST)"
-                },
-                "edata.eks.avg_num_launches": {
-                  "to": "Average num. of Genie launches per day"
-                },
-                "edata.eks.num_contents": {
-                  "to": "No. of content pieces available on device"
-                },
-                "edata.eks.play_start_time": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Time stamp when the device first started game play(IST)"
-                },
-                "edata.eks.end_time": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Time stamp when the device was last used(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "edata.eks.num_days": {
-                  "to": "Number of days since Genie installed before the timestamp indicating the last use"
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.mean_play_time": {
-                  "to": "Average play time on device"
-                },
-                "edata.eks.mean_play_time_interval": {
-                  "to": "Average time interval between game play on the device"
-                },
-                "edata.eks.last_played_on": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Time stamp when the device was last used for game play(IST)"
-                },
-                "edata.eks.avg_time": {
-                  "to": "Average time on Genie per day"
-                },
-                "edata.eks.last_played_content": {
-                  "to": "Last played content on device"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "dimensions.did": {
-                  "to": "Device ID"
-                },
-                "edata.eks.num_sessions": {
-                  "to": "Total number of sessions from the device"
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_GENIE_SESSION_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/cus/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "genieTag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.content": {
-                  "to": "Content IDs of content used in Genie Session"
-                },
-                "edata.eks.timeSpent": {
-                  "to": "Time spent on Genie Session\t"
-                },
-                "uid": {
-                  "to": "uid"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation time(IST)"
-                },
-                "dimensions.group_user": {
-                  "to": "Is Group user?"
-                },
-                "edata.eks.contentCount": {
-                  "to": "Number of content pieces used"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "edata.eks.time_stamp": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Session End time stamp(IST)"
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "tags": {
-                  "to": "Genie tags"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "dimensions.anonymous_user": {
-                  "to": "Logged in user?"
-                },
-                "dimensions.did": {
-                  "to": "Device ID"
-                },
-                "mid": {
-                  "to": "Unique Genie Session ID"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          }
-        }
-      },
-      "eks-consumption-metrics": {
-        "events": [
-          "ME_CONTENT_USAGE_METRICS",
-          "ME_ITEM_USAGE_METRICS",
-          "ME_GENIE_USAGE_METRICS",
-          "ME_CONTENT_SNAPSHOT_METRICS",
-          "ME_CONCEPT_SNAPSHOT_METRICS",
-          "ME_ASSET_SNAPSHOT_METRICS"
-        ],
-        "eventConfig": {
-          "ME_CONTENT_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_content_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "dimensions.tag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.content_id": {
-                  "to": "Content ID"
-                },
-                "dimensions.tag": {
-                  "to": "tag"
-                },
-                "edata.eks.m_publish_date": {
-                  "to": "Date when the content is published"
-                },
-                "edata.eks.m_total_ts": {
-                  "to": "Total time spent"
-                },
-                "edata.eks.m_total_sessions": {
-                  "to": "Total number of sessions"
-                },
-                "edata.eks.m_avg_ts_session": {
-                  "to": "Average timespent per session"
-                },
-                "edata.eks.m_total_interactions": {
-                  "to": "Total interactions count"
-                },
-                "edata.eks.m_avg_interactions_min": {
-                  "to": "Average interactions per minute"
-                },
-                "edata.eks.m_total_devices": {
-                  "to": "Total number of devices"
-                },
-                "edata.eks.m_avg_sess_device": {
-                  "to": "Average sessions per device"
-                }
-              }
-            }
-          },
-          "ME_ITEM_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_item_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "dimensions.tag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.tag": {
-                  "to": "tag"
-                },
-                "dimensions.content_id": {
-                  "to": "Content ID"
-                },
-                "dimensions.item_id": {
-                  "to": "item id"
-                },
-                "edata.eks.m_total_ts": {
-                  "to": "Total time spent"
-                },
-                "edata.eks.m_total_count": {
-                  "to": "Total response count"
-                },
-                "edata.eks.m_correct_res_count": {
-                  "to": "Correct response count"
-                },
-                "edata.eks.m_inc_res_count": {
-                  "to": "Incorrect response count"
-                },
-                "edata.eks.m_correct_res": {
-                  "hidden": true
-                },
-                "edata.eks.m_incorrect_res": {
-                  "hidden": true
-                },
-                "edata.eks.m_top5_incorrect_res": {
-                  "to": "Top 5 Incorrect response"
-                },
-                "edata.eks.m_avg_ts": {
-                  "to": "Average time spent"
-                },
-                "edata.eks.m_top5_misconception_res": {
-                  "to": "Top 5 Misconception response"
-                }
-              }
-            }
-          },
-          "ME_GENIE_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_genie_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "tags": {
-                "name": "dimensions.tag",
-                "operator": "IN"
-              },
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.tag": {
-                  "to": "Device Tag"
-                },
-                "edata.eks.m_total_sessions": {
-                  "to": "Total number of sessions"
-                },
-                "edata.eks.m_total_ts": {
-                  "to": "Total time spent"
-                },
-                "edata.eks.m_avg_ts_session": {
-                  "to": "Average sessions per device"
-                },
-                "edata.eks.m_contents": {
-                  "hidden": true
-                },
-                "edata.eks.m_total_devices": {
-                  "to": "Total number of devices"
-                }
-              }
-            }
-          },
-          "ME_CONTENT_SNAPSHOT_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_content_snapshot_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          },
-          "ME_CONCEPT_SNAPSHOT_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_concept_snapshot_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          },
-          "ME_ASSET_SNAPSHOT_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-consumption-metrics/me_asset_snapshot_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          }
-        }
-      },
-      "eks-creation-raw": {
-        "events": [
-          "eks-creation-raw"
-        ],
-        "eventConfig": {
-          "eks-creation-raw": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "unique/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "context.channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "context.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          }
-        }
-      },
-      "eks-creation-summary": {
-        "events": [
-          "ME_APP_SESSION_SUMMARY",
-          "ME_CE_SESSION_SUMMARY",
-          "ME_TEXTBOOK_SESSION_SUMMARY",
-          "ME_APP_USAGE_SUMMARY",
-          "ME_CE_USAGE_SUMMARY",
-          "ME_TEXTBOOK_USAGE_SUMMARY",
-          "ME_AUTHOR_USAGE_SUMMARY",
-          "ME_PUBLISH_PIPELINE_SUMMARY"
-        ],
-        "eventConfig": {
-          "ME_APP_SESSION_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/app-ss/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "edata.eks.interact_events_per_min": {
-                  "to": "Average ineractions per minute"
-                },
-                "edata.eks.events_summary": {
-                  "to": "edata.eks.events_summary"
-                },
-                "dimensions.app_id": {
-                  "to": "Event Source"
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.start_time": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Start Time(IST)"
-                },
-                "uid": {
-                  "to": "User ID"
-                },
-                "edata.eks.page_summary": {
-                  "to": "Pages visited Summary"
-                },
-                "ets": {
-                  "hidden": true
-                },
-                "edata.eks.page_views_count": {
-                  "to": "Total pageviews in Session"
-                },
-                "edata.eks.end_time": {
-                  "hidden": true
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.ce_visits": {
-                  "to": "Number of Authoring tool visits"
-                },
-                "dimensions.sid": {
-                  "to": "Session ID"
-                },
-                "edata.eks.time_diff": {
-                  "to": "Session length "
-                },
-                "edata.eks.interact_events_count": {
-                  "to": "Number of Interactions"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "edata.eks.time_spent": {
-                  "to": "Session length"
-                },
-                "dimensions.anonymous_user": {
-                  "to": "User Logged In"
-                },
-                "mid": {
-                  "to": "mid"
-                },
-                "edata.eks.env_summary": {
-                  "to": "Usage summary"
-                },
-                "edata.eks.first_visit": {
-                  "to": "First Time Visitor"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_CE_SESSION_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/ce-ss/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_TEXTBOOK_SESSION_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/textbook-ss/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_APP_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/app-usage/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "edata.eks.avg_ts_session": {
-                  "to": "Average session duration per day"
-                },
-                "context.granularity": {
-                  "to": "Granularity"
-                },
-                "edata.eks.anon_total_sessions": {
-                  "to": "Time spent by non-registered users"
-                },
-                "edata.eks.unique_users": {
-                  "to": "Unique users list"
-                },
-                "dimensions.app_id": {
-                  "to": "Source"
-                },
-                "edata.eks.unique_users_count": {
-                  "to": "Unique users count"
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "edata.eks.total_pageviews_count": {
-                  "to": "Total Page Views per day"
-                },
-                "uid": {
-                  "to": "uid"
-                },
-                "edata.eks.ce_total_sessions": {
-                  "to": "Number of sessions where creation happened"
-                },
-                "edata.eks.anon_avg_ts_session": {
-                  "to": "Avg. session duration for non-registered users per day"
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Generation Time(IST)"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Total time spent by visitors on Portal"
-                },
-                "edata.eks.ce_percent_sessions": {
-                  "to": "Percentage of visitors creating content"
-                },
-                "dimensions.author_id": {
-                  "to": "Author ID"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.new_user_count": {
-                  "to": "Number of new users"
-                },
-                "syncts": {
-                  "to": "Event sync time(IST)",
-                  "mapFunc": "timestampToDateTime"
-                },
-                "edata.eks.percent_new_users_count": {
-                  "to": "Percentage of new visitors per day"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "edata.eks.avg_pageviews": {
-                  "to": "Avg. pages per visit per day"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "Total number of sessions"
-                },
-                "edata.eks.anon_total_ts": {
-                  "to": "Total time spent by non-registered users on Portal"
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "dimensions.period"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_CE_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/ce-usage/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_TEXTBOOK_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/textbook-usage/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": false,
-              "columnMappings": {}
-            }
-          },
-          "ME_AUTHOR_USAGE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/author-usage-summary/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "uid": {
-                  "to": "Author ID"
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Sync time(IST)"
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "edata.eks.avg_ts_session": {
-                  "to": "Total number of seesion per day"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Total time spent by author"
-                },
-                "edata.eks.ce_total_ts": {
-                  "to": "Total time spent on content editor by author"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "No of sessions per Author"
-                },
-                "context.granularity": {
-                  "hidden": true
-                },
-                "edata.eks.ce_percent_ts": {
-                  "to": "Percentage of time spent browsing vs creating"
-                },
-                "edata.eks.ce_percent_sessions": {
-                  "to": "Percentage of sessions in which users log into content editor"
-                },
-                "edata.eks.ce_total_visits": {
-                  "to": "No of Content Editor visits per author"
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          },
-          "ME_PUBLISH_PIPELINE_SUMMARY": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "derived/publish-pipeline-summ/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "context.granularity": {
-                  "hidden": true
-                },
-                "context.date_range.from": {
-                  "hidden": true
-                },
-                "uid": {
-                  "hidden": true
-                },
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "ver": {
-                  "hidden": true
-                },
-                "context.pdata.id": {
-                  "hidden": true
-                },
-                "edata.eks.publish_pipeline_summary": {
-                  "to": "edata.eks.publish_pipeline_summary"
-                },
-                "syncts": {
-                  "to": "Event sync time(IST)",
-                  "mapFunc": "timestampToDateTime"
-                },
-                "context.pdata.ver": {
-                  "hidden": true
-                },
-                "context.pdata.model": {
-                  "hidden": true
-                },
-                "mid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Date"
-                },
-                "context.date_range.to": {
-                  "hidden": true
-                }
-              }
-            }
-          }
-        }
-      },
-      "eks-creation-metrics": {
-        "events": [
-          "ME_APP_USAGE_METRICS",
-          "ME_CE_USAGE_METRICS",
-          "ME_TEXTBOOK_CREATION_METRICS",
-          "ME_TEXTBOOK_SNAPSHOT_METRICS",
-          "ME_AUTHOR_USAGE_METRICS"
-        ],
-        "eventConfig": {
-          "ME_APP_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-creation-metrics/me_app_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.author_id": {
-                  "to": "Author ID / All"
-                },
-                "dimensions.app_id": {
-                  "to": "App (EkStep portal currently)"
-                },
-                "edata.eks.anon_total_sessions": {
-                  "to": "Total anonymous sessions on portal"
-                },
-                "edata.eks.anon_total_ts": {
-                  "to": "Tota time spent (Anonymous) on portal"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "Total number of portal sessions"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Total time spent on portal"
-                },
-                "edata.eks.ce_total_sessions": {
-                  "to": "Total content editor sessions"
-                },
-                "edata.eks.ce_percent_sessions": {
-                  "to": "% of total sessions that are content editor sessions"
-                },
-                "edata.eks.total_pageviews_count": {
-                  "to": "Total pageviews on portal"
-                },
-                "edata.eks.unique_users": {
-                  "hidden": true
-                },
-                "edata.eks.unique_users_count": {
-                  "to": "Count of unique users on portal"
-                },
-                "edata.eks.avg_pageviews": {
-                  "to": "Avg. number of pageviews per session for specified period"
-                },
-                "edata.eks.avg_ts_session": {
-                  "to": "Avg. session length (time spent) per session for specified period"
-                },
-                "edata.eks.anon_avg_ts_session": {
-                  "to": "Avg. Anonymous session length (time spent) per session for specified period"
-                },
-                "edata.eks.new_user_count": {
-                  "to": "No. of new users on the app (portal) for the given time period"
-                },
-                "edata.eks.percent_new_users_count": {
-                  "to": "% of new users for the given time period"
-                }
-              }
-            }
-          },
-          "ME_CE_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-creation-metrics/me_ce_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": "Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.content_id": {
-                  "to": "Content ID"
-                },
-                "edata.eks.unique_users_count": {
-                  "to": "Number of users who accessed the content piece via the editor"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "Total number of sessions that accessed the content piece via the editor"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Time spent on the content piece in the content editor"
-                },
-                "edata.eks.avg_ts_session": {
-                  "to": "Average session time on the editor"
-                },
-                "edata.eks.update_date": {
-                  "to": "Last updated timestamp of the content piece"
-                }
-              }
-            }
-          },
-          "ME_TEXTBOOK_CREATION_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-creation-metrics/me_textbook_creation_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          },
-          "ME_TEXTBOOK_SNAPSHOT_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-creation-metrics/me_textbook_snapshot_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            }
-          },
-          "ME_AUTHOR_USAGE_METRICS": {
-            "searchType": "azure",
-            "fetchConfig": {
-              "params": {
-                "bucket": "{{ bucket }}",
-                "prefix": "eks-creation-metrics/me_author_usage_metrics/"
-              }
-            },
-            "filterMapping": {
-              "channel": {
-                "name": "channel",
-                "operator": "EQ"
-              },
-              "app_id": {
-                "name": "dimensions.pdata.id",
-                "operator": "EQ"
-              }
-            },
-            "csvConfig": {
-              "auto_extract_column_names": true,
-              "columnMappings": {
-                "ets": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event generation Time(IST)"
-                },
-                "syncts": {
-                  "mapFunc": "timestampToDateTime",
-                  "to": "Event sync time(IST)"
-                },
-                "eid": {
-                  "hidden": true
-                },
-                "dimensions.period": {
-                  "to": " Time period of the summary. For ex: Day or Week or Month represented by a number"
-                },
-                "dimensions.author_id": {
-                  "to": "Author ID /All"
-                },
-                "edata.eks.total_sessions": {
-                  "to": "Total number of portal sessions by author"
-                },
-                "edata.eks.total_ts": {
-                  "to": "Total time spent on portal by author"
-                },
-                "edata.eks.total_ce_ts": {
-                  "to": "Total time spent on content editor by author"
-                },
-                "edata.eks.total_ce_visit": {
-                  "to": "Number of content editor visits"
-                },
-                "edata.eks.percent_ce_sessions": {
-                  "to": "% of sessions that were spent on the editor"
-                },
-                "edata.eks.avg_ts_session": {
-                  "to": "Average time spent by author on portal per session"
-                },
-                "edata.eks.percent_ce_ts": {
-                  "to": "% time spent by author on content editor"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
   "wfs": {
     "search": {
       "type": "azure",
       "queries": [
         {
           "bucket": "{{ bucket }}",
-          "prefix": "unique/",
+          "prefix": "unique/raw/",
           "endDate": "$(date --date yesterday '+%Y-%m-%d')",
           "delta": 0
         }
@@ -1891,167 +42,6 @@
     "appName": "Workflow Summarizer",
     "deviceMapping": true
   },
-  "wfus": {
-    "search": {
-      "type": "azure",
-      "queries": [
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "{{ job_manager_tmp_dir }}/wfs/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0,
-          "folder": "true"
-        }
-      ]
-    },
-    "model": "org.ekstep.analytics.model.WorkflowUsageSummary",
-    "modelParams": {
-      "apiVersion": "v2"
-    },
-    "output": [
-      {
-        "to": "azure",
-        "params": {
-          "bucket": "{{ bucket }}",
-          "key": "{{ job_manager_tmp_dir }}/wfus/$(date --date yesterday '+%Y-%m-%d')"
-        }
-      },
-      {
-        "to": "kafka",
-        "params": {
-          "brokerList": "{{ brokerlist }}",
-          "topic": "{{ topic }}"
-        }
-      }
-    ],
-    "parallelization": 8,
-    "appName": "Workflow Usage Summarizer",
-    "deviceMapping": true
-  },
-  "wfu": {
-    "search": {
-      "type": "azure",
-      "queries": [
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "{{ job_manager_tmp_dir }}/wfus/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0,
-          "folder": "true"
-        }
-      ]
-    },
-    "appName": "Workflow Usage Updater",
-    "parallelization": 8,
-    "model": "org.ekstep.analytics.updater.UpdateWorkFlowUsageDB",
-    "modelParams": {},
-    "output": [
-      {
-        "to": "console",
-        "params": {
-          "printEvent": false
-        }
-      }
-    ]
-  },
-  "ds": {
-    "search": {
-      "type": "azure",
-      "queries": [
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "unique/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0
-        },
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "{{ job_manager_tmp_dir }}/wfs/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0,
-          "folder": "true"
-        }
-      ]
-    },
-    "model": "org.ekstep.analytics.model.DeviceSummaryModel",
-    "modelParams": {
-      "apiVersion": "v2"
-    },
-    "output": [
-      {
-        "to": "azure",
-        "params": {
-          "bucket": "{{ bucket }}",
-          "key": "{{ job_manager_tmp_dir }}/device-summary/$(date --date yesterday '+%Y-%m-%d')"
-        }
-      },
-      {
-        "to": "kafka",
-        "params": {
-          "brokerList": "{{ brokerlist }}",
-          "topic": "{{ topic }}"
-        }
-      }
-    ],
-    "parallelization": 12,
-    "appName": "Device Summarizer",
-    "deviceMapping": false
-  },
-  "dpu": {
-    "search": {
-      "type": "azure",
-      "queries": [
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "{{ job_manager_tmp_dir }}/device-summary/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0,
-          "folder": "true"
-        }
-      ]
-    },
-    "model": "org.ekstep.analytics.model.UpdateDeviceProfileDB",
-    "modelParams": {
-      "apiVersion": "v2"
-    },
-    "output": [
-      {
-        "to": "console",
-        "params": {
-          "printEvent": false
-        }
-      }
-    ],
-    "parallelization": 8,
-    "appName": "Device Profile Updater",
-    "deviceMapping": false
-  },
-  "workflow-usage-metrics": {
-    "search": {
-      "type": "azure",
-      "queries": [
-        {
-          "bucket": "{{ bucket }}",
-          "prefix": "{{ job_manager_tmp_dir }}/wfus/",
-          "endDate": "$(date --date yesterday '+%Y-%m-%d')",
-          "delta": 0,
-          "folder": "true"
-        }
-      ]
-    },
-    "model": "org.ekstep.analytics.updater.UpdateWorkFlowUsageMetricsMode",
-    "output": [
-      {
-        "to": "console",
-        "params": {
-          "printEvent": false
-        }
-      }
-    ],
-    "parallelization": 10,
-    "appName": "Workflow Usage Metrics Updater",
-    "deviceMapping": false
-  },
   "video-streaming": {
     "search": {
       "type": "azure"
@@ -2072,134 +62,6 @@
     "appName": "Video Streaming Job",
     "deviceMapping": false
   },
-  "course-dashboard-metrics": {
-    "search": {
-      "type": "none"
-    },
-    "model": "org.ekstep.analytics.updater.UpdateCourseMetrics",
-    "modelParams": {
-        "druidConfig": {
-          "queryType": "groupBy",
-          "dataSource": "content-model-snapshot",
-          "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00",
-          "granularity": "all",
-          "aggregations": [
-            {
-              "name": "count",
-              "type": "count",
-              "fieldName": "count"
-            }
-          ],
-          "dimensions": [
-            {
-              "fieldName": "identifier",
-              "aliasName": "identifier"
-            },
-            {
-              "fieldName": "channel",
-              "aliasName": "channel"
-            }
-          ],
-          "filters": [
-            {
-              "type": "equals",
-              "dimension": "contentType",
-              "value": "Course"
-            },
-            {
-              "type": "isnotnull",
-              "dimension": "identifier",
-              "value": ""
-            },
-            {
-              "type": "isnotnull",
-              "dimension": "channel",
-              "value": ""
-            }
-          ],
-          "descending": "false"
-        },
-      "fromDate": "$(date --date yesterday '+%Y-%m-%d')",
-      "toDate": "$(date --date yesterday '+%Y-%m-%d')",
-      "sparkCassandraConnectionHost": "{{core_cassandra_host}}",
-      "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}"
-    },
-    "output": [
-      {
-        "to": "console",
-        "params": {
-          "printEvent": false
-        }
-      }
-    ],
-    "parallelization": 8,
-    "appName": "Course Dashboard Metrics",
-    "deviceMapping": false
-  },
-  "assessment-dashboard-metrics": {
-    "search": {
-      "type": "none"
-    },
-    "model": "org.sunbird.analytics.job.report.AssessmentMetricsJob",
-    "modelParams": {
-        "druidConfig": {
-          "queryType": "groupBy",
-          "dataSource": "content-model-snapshot",
-          "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00",
-          "granularity": "all",
-          "aggregations": [
-            {
-              "name": "count",
-              "type": "count",
-              "fieldName": "count"
-            }
-          ],
-          "dimensions": [
-            {
-              "fieldName": "identifier",
-              "aliasName": "identifier"
-            },
-            {
-              "fieldName": "channel",
-              "aliasName": "channel"
-            }
-          ],
-          "filters": [
-            {
-              "type": "equals",
-              "dimension": "contentType",
-              "value": "Course"
-            },
-            {
-              "type": "isnotnull",
-              "dimension": "identifier",
-              "value": ""
-            },
-            {
-              "type": "isnotnull",
-              "dimension": "channel",
-              "value": ""
-            }
-          ],
-          "descending": "false"
-        },
-      "fromDate": "$(date --date yesterday '+%Y-%m-%d')",
-      "toDate": "$(date --date yesterday '+%Y-%m-%d')",
-      "sparkCassandraConnectionHost": "{{core_cassandra_host}}",
-      "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}"
-    },
-    "output": [
-      {
-        "to": "console",
-        "params": {
-          "printEvent": false
-        }
-      }
-    ],
-    "parallelization": 8,
-    "appName": "Assessment Dashboard Metrics",
-    "deviceMapping": false
-  },
   "admin-user-reports": {
     "search": {
       "type": "none"
@@ -2278,51 +140,11 @@
       "brokerList": "{{ brokerlist }}",
       "topic": "{{ analytics_metrics_topic }}",
       "model": [
-        {
-          "model": "DeviceSummaryModel",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "UpdateDeviceProfileDB",
-          "category": "consumption",
-          "input_dependency": "DeviceSummaryModel"
-        },
-        {
-          "model": "DataExhaustJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "UpdateWorkFlowUsageDB",
-          "category": "consumption",
-          "input_dependency": "WorkFlowUsageSummaryModel"
-        },
         {
           "model": "WorkFlowSummaryModel",
           "category": "consumption",
           "input_dependency": "None"
         },
-        {
-          "model": "WorkFlowUsageSummaryModel",
-          "category": "consumption",
-          "input_dependency": "WorkFlowSummaryModel"
-        },
-        {
-          "model": "UpdatePortalMetrics",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "UpdateWorkFlowUsageMetricsModel",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "CourseMetricsJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
         {
           "model": "UpdateContentRating",
           "category": "consumption",
@@ -2333,36 +155,11 @@
           "category": "consumption",
           "input_dependency": "None"
         },
-        {
-          "model": "DistrictMonthlyJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "DesktopConsumptionDailyMetricsJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "DistrictWeeklyJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
-        {
-          "model": "ConsumptionDailyMetricsJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
         {
           "model": "MetricsAuditJob",
           "category": "consumption",
           "input_dependency": "None"
         },
-        {
-          "model": "AssessmentMetricsJob",
-          "category": "consumption",
-          "input_dependency": "None"
-        },
         {
           "model": "StateAdminReportJob",
           "category": "consumption",
@@ -2679,79 +476,6 @@
     "appName": "TPD Course Consumption Metrics Model",
     "deviceMapping": false
   },
-  "textbook-progress-report": {
-      "search": {
-        "type": "none"
-      },
-      "model": "org.ekstep.analytics.model.report.TextBookProgressModel",
-      "modelParams": {
-        "reportConfig": {
-          "id": "content_progress_metrics",
-          "metrics": [],
-          "labels": {
-            "board": "Board",
-            "medium": "Medium",
-            "gradeLevel": "Grade",
-            "subject": "Subject",
-            "resourceType": "Content Type",
-            "totalContent": "Total Contents",
-            "live": "Live",
-            "review": "Review",
-            "draft": "Draft",
-            "unlisted": "Limited Sharing",
-            "application_ecml": "Created on Diksha",
-            "video_youtube": "YouTube Content",
-            "video_mp4": "Uploaded Videos",
-            "application_pdf": "Text Content",
-            "application_html": "Uploaded Interactive Content",
-            "identifier": "Content ID",
-            "creator": "Created By",
-            "createdOn": "Creation Date",
-            "lastPublishDate": "Last Publish Date",
-            "status": "Status",
-            "pkgVersion": "Number of times Published",
-            "lastPublishedOn": "Pending in current status since",
-            "pendingInCurrentStatus": "Pending in current status since"
-          },
-          "output": [{
-              "type": "csv",
-              "dims": []
-            }],
-          "mergeConfig": {
-          "frequency": "WEEK",
-          "basePath": "{{ spark_output_temp_dir }}",
-          "rollup": 0,
-          "reportPath": "content_progress_metrics.csv",
-          "postContainer":"{{ reports_container }}"
-        }
-        },
-        "filter": {
-          "tenantId": "",
-          "slugName": ""
-        },
-        "store": "azure",
-        "format": "csv",
-        "key": "druid-reports/",
-        "filePath": "druid-reports/",
-        "container": "{{ bucket }}",
-        "folderPrefix": [
-          "slug",
-          "reportName"
-        ],
-        "sparkCassandraConnectionHost": "{{core_cassandra_host}}"
-      },
-      "output": [
-        {
-          "to": "console",
-          "params": {
-            "printEvent": false
-          }
-        }
-      ],
-      "parallelization": 8,
-      "appName": "Textbook Progress Metrics Model",
-      "deviceMapping": false
-    },
   "audit-metrics-report": {
       "search": {
         "type": "none"
diff --git a/ansible/roles/data-products-deploy/templates/model-dock-config.j2 b/ansible/roles/data-products-deploy/templates/model-dock-config.j2
new file mode 100644
index 0000000000..20d82dbfb5
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/model-dock-config.j2
@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+config() {
+	bucket={{ bucket }}
+	brokerList={{ brokerlist }}
+	zookeeper={{ zookeeper }}
+	job_topic={{ analytics_job_queue_topic }}
+	topic={{ topic }}
+	sparkCassandraConnectionHost="{{ lp_cassandra_host  }}"
+	sunbirdPlatformCassandraHost="{{ core_cassandra_host }}"
+	reportPostContainer="{{ reports_container }}"
+	druidRollupHost="{{ druid_rollup_cluster_ingestion_task_url }}"
+	
+	if [ -z "$2" ]; then endDate=$(date --date yesterday "+%Y-%m-%d"); else endDate=$2; fi
+	if [ ! -z "$3" ]; then inputBucket=$3; fi
+	case "$1" in
+		"content-details")
+		echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"azure","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}'
+		;;
+		"sourcing-summary-report")
+		echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "azure", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}'
+		;;
+		"funnel-report")
+		echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "azure","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}'
+		;;
+		"sourcing-metrics")
+		echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "azure","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}'
+		;;
+        "*")
+		echo "Unknown model code"
+      	exit 1 # Command to come out of the program with status 1
+      	;;
+	esac
+}
diff --git a/ansible/roles/data-products-deploy/templates/monitor-dp.j2 b/ansible/roles/data-products-deploy/templates/monitor-dp.j2
deleted file mode 100644
index a111aa4649..0000000000
--- a/ansible/roles/data-products-deploy/templates/monitor-dp.j2
+++ /dev/null
@@ -1,150 +0,0 @@
-#!/bin/bash
-
-log_file=$1
-
-total_jobs=`grep "BE_JOB_END" $log_file`
-comp_jobs=`grep "status\":\"SUCCESS" <<< "$total_jobs"`
-failed_jobs=`grep "status\":\"FAILED" <<< "$total_jobs"`
-
-today=$(date "+%Y-%m-%d")
-
-comp_num=0
-failed_num=0
-total_events=0
-total_time=0
-
-file_content="Model,Job Status,Input Events Count, Output Events Count,Time Taken(in Seconds),Date\n"
-job_status="COMPLETED"
-# Declaring Arrays
-declare -A input
-declare -A output
-warnings=""
-
-if [ "$comp_jobs" != "" ]; then
-	comp_num=`echo "$comp_jobs" | wc -l | bc`
-	while read -r line
-	do	
-	    model=`sed 's/.*model":"\(.*\)","ver.*/\1/' <<< "$line" | sed -e "s/org.ekstep.analytics.model.//g" | sed -e "s/org.ekstep.analytics.updater.//g"`
-		#message=`sed 's/.*message":"\(.*\)","class.*/\1/' <<< "$line"`
-
-		eventsIn=`sed 's/.*","inputEvents":\(.*\),"outputEvents":.*/\1/' <<< "$line"`
-		sub="BE_JOB_END"
-		if [ "${eventsIn/$sub}" != "$eventsIn" ] ; then
-  			eventsIn=0
-  		fi
-		events=`sed 's/.*"outputEvents":\(.*\),"timeTaken".*/\1/' <<< "$line"`
-		if [ "${events/$sub}" != "$events" ] ; then
-  			events=0
-  		fi
-		total_events=$((total_events + events))
-		time_taken=`sed 's/.*"timeTaken":\(.*\)},".*/\1/' <<< "$line"`
-		total_time=`echo $total_time + $time_taken | bc`		
-		event_date=`sed 's/.*{"date":"\(.*\)","inputEvents":.*/\1/' <<< "$line"`
-		if [ "${event_date/$sub}" != "$event_date" ] ; then
-  			event_date=""
-  		fi
-  		input+=( [$model]=$eventsIn )
-        output+=( [$model]=$events )
-		file_content+="$model,$job_status,$eventsIn,$events,$time_taken,$event_date\n"
-	done <<< "$comp_jobs"
-fi
-
-events="0"
-time_taken="0"
-job_status="FAILED"
-event_date=""
-
-if [ "$failed_jobs" != "" ]; then
-	failed_num=`echo "$failed_jobs" | wc -l | bc`
-	while read -r line
-	do
-		model=`sed 's/.*model":"\(.*\)","ver.*/\1/' <<< "$line" | sed -e "s/org.ekstep.analytics.model.//g" | sed -e "s/org.ekstep.analytics.updater.//g"`
-		#message=`grep -Po '(?<="localizedMessage":")[^"]*' <<< "$line"`
-		sub="BE_JOB_END"
-		eventsIn=`sed 's/.*","inputEvents":\(.*\)},".*/\1/' <<< "$line"`
-		if [ "${eventsIn/$sub}" != "$eventsIn" ] ; then
-  			eventsIn=0
-  		fi
-		event_date=`sed 's/.*"data":{"date":"\(.*\)","inputEvents".*/\1/' <<< "$line"`
-		if [ "${event_date/$sub}" != "$event_date" ] ; then
-  			event_date=""
-  		fi
-  		echo "$event_date"
-  		input+=( [$model]=$eventsIn )
-        output+=( [$model]=$events )
-		file_content+="$model,$job_status,$eventsIn,$events,$time_taken,$event_date\n"
-	done <<< "$failed_jobs"
-fi
-
-if [[ ${output["LearnerSessionSummaryModel"]} != ${input["ItemUsageSummaryModel"]} ]]; then
-	warnings+="Output of LearnerSessionSummaryModel NOT EQUALS to Input of ItemUsageSummaryModel\n"   
-fi
-
-if [[ ${output["GenieLaunchSummaryModel"]} != ${input["GenieUsageSummaryModel"]} ]]; then
-	warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of GenieUsageSummaryModel\n"    
-fi
-
-if [[ ${output["GenieFunnelModel"]} != ${input["GenieFunnelAggregatorModel"]} ]]; then
-	warnings+="Output of GenieFunnelModel NOT EQUALS to Input of GenieFunnelAggregatorModel\n"   
-fi
-
-if [[ ${output["GenieLaunchSummaryModel"]} != ${input["GenieStageSummaryModel"]} ]]; then
-	warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of GenieStageSummaryModel\n"   
-fi
-
-if [[ ${output["ItemUsageSummaryModel"]} != ${input["ItemSummaryModel"]} ]]; then
-	warnings+="Output of ItemUsageSummaryModel NOT EQUALS to Input of ItemSummaryModel\n"    
-fi
-
-if [[ ${output["LearnerSessionSummaryModel"]} != ${input["DeviceContentUsageSummaryModel"]} ]]; then
-	warnings+="Output of LearnerSessionSummaryModel NOT EQUALS to Input of DeviceContentUsageSummaryModel\n"    
-fi
-
-if [[ ${output["GenieLaunchSummaryModel"]} != ${input["DeviceUsageSummaryModel"]} ]]; then
-	warnings+="Output of GenieLaunchSummaryModel NOT EQUALS to Input of DeviceUsageSummaryModel\n"    
-fi
-
-if [[ ${output["GenieUsageSummaryModel"]} != ${input["UpdateGenieUsageDB"]} ]]; then
-	warnings+="Output of GenieUsageSummaryModel NOT EQUALS to Input of UpdateGenieUsageDB\n"    
-fi
-
-if [[ ${output["ItemSummaryModel"]} != ${input["UpdateItemSummaryDB"]} ]]; then
-	warnings+="Output of ItemSummaryModel NOT EQUALS to Input of UpdateItemSummaryDB\n"    
-fi
-
-if [[ ${output["ContentPopularitySummaryModel"]} != ${input["UpdateContentPopularityDB"]} ]]; then
-	warnings+="Output of ContentPopularitySummaryModel NOT EQUALS to Input of UpdateContentPopularityDB\n"    
-fi
-
-if [[ ${output["ContentUsageSummaryModel"]} != ${input["UpdateContentUsageDB"]} ]]; then
-	warnings+="Output of ContentUsageSummaryModel NOT EQUALS to Input of UpdateContentUsageDB\n"   
-fi
-
-echo -e $file_content > {{ analytics.home }}/scripts/monitor-data/dp-monitor-$today.csv
-
-
-echo "-------- Status Report --------"
-echo "Number of Completed Jobs: $comp_num"
-echo "Number of failed Jobs: $failed_num"
-echo "Total time taken: $total_time"
-echo "Total events generated: $total_events"
-
-
-if [[("$failed_num" > 0 && "$warnings" == "") ]]; then
-
-data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nError: ```'"Job Failed"'```", "icon_emoji": ":ghost:"}'
-
-elif [[("$warnings" == "" && "$failed_num" == 0)]]; then
-
-data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nMessage: ```'"Job Run Completed Successfully"'```", "icon_emoji": ":ghost:"}'
-
-elif [[("$warnings" != "" && "$failed_num" == 0)]]; then
-
-data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nWarnings: ```'$warnings'``` ", "icon_emoji": ":ghost:"}'
-
-else
-
-data='{"channel": "{{ channel }}", "username": "{{ dp_username }}", "text":"*Jobs | Monitoring Report | '$today'*\nNumber of Completed Jobs: `'$comp_num'` \nNumber of failed Jobs: `'$failed_num'` \nTotal time taken: `'$total_time'`\nTotal events generated: `'$total_events'`\n\nDetailed Report:\n```'$file_content'```\n\nError: ```'"Job Failed"'```\n\nWarnings:```'$warnings'```", "icon_emoji": ":ghost:"}'
-fi
-
-curl -X POST -H 'Content-Type: application/json' --data "$data" https://hooks.slack.com/services/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
diff --git a/ansible/roles/data-products-deploy/templates/replay-job.j2 b/ansible/roles/data-products-deploy/templates/replay-job.j2
index 72c9dcfcfe..3a6c969b7b 100644
--- a/ansible/roles/data-products-deploy/templates/replay-job.j2
+++ b/ansible/roles/data-products-deploy/templates/replay-job.j2
@@ -7,6 +7,8 @@ cd {{ analytics.home }}/scripts
 source model-config.sh
 source replay-utils.sh
 
+libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0"
+
 if [ "$1" == "telemetry-replay" ]
     then
     if [ ! $# -eq 5 ]
@@ -15,6 +17,17 @@ if [ "$1" == "telemetry-replay" ]
         exit
     fi
 fi
+
+get_report_job_model_name(){
+    case "$1" in
+        "assessment-correction") echo 'org.sunbird.analytics.job.report.AssessmentCorrectionJob'
+        ;;
+        *) echo $1
+        ;;
+    esac
+}
+
+if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi
 if [ -z "$job_config" ]; then job_config=$(config $1 '__endDate__' $4 $5); fi
 start_date=$2
 end_date=$3
@@ -33,7 +46,8 @@ if [ $? == 0 ]
  	then
   	echo "Backup completed Successfully..." >> "$DP_LOGS/$end_date-$1-replay.log"
   	echo "Running the $1 job replay..." >> "$DP_LOGS/$end_date-$1-replay.log"
-  	$SPARK_HOME/bin/spark-submit --master local[*] --jars $MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar --class org.ekstep.analytics.job.ReplaySupervisor $MODELS_HOME/batch-models-2.0.jar --model "$1" --fromDate "$start_date" --toDate "$end_date" --config "$job_config" >> "$DP_LOGS/$end_date-$1-replay.log"
+  	echo "Job modelName - $job_id" >> "$DP_LOGS/$end_date-$1-replay.log"
+  	$SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.12-2.5.0.jar --class org.ekstep.analytics.job.ReplaySupervisor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --fromDate "$start_date" --toDate "$end_date" --config "$job_config" >> "$DP_LOGS/$end_date-$1-replay.log"
 else
   	echo "Unable to take backup" >> "$DP_LOGS/$end_date-$1-replay.log"
 fi
diff --git a/ansible/roles/data-products-deploy/templates/run-dock-job.j2 b/ansible/roles/data-products-deploy/templates/run-dock-job.j2
new file mode 100644
index 0000000000..e6f1cdf9ad
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/run-dock-job.j2
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+export SPARK_HOME={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7
+export MODELS_HOME={{ analytics.home }}/models-{{ model_version }}
+export DP_LOGS={{ analytics.home }}/logs/data-products
+## Job to run daily
+cd {{ analytics.home }}/scripts
+source model-dock-config.sh
+today=$(date "+%Y-%m-%d")
+
+libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0"
+file_path="dock-{{ env }}.conf"
+
+get_report_job_model_name(){
+	case "$1" in
+		"funnel-report") echo 'org.sunbird.analytics.sourcing.FunnelReport'
+		;;
+		"sourcing-summary-report") echo 'org.sunbird.analytics.sourcing.SourcingSummaryReport'
+		;;
+		"sourcing-metrics") echo 'org.sunbird.analytics.sourcing.SourcingMetrics'
+		;;
+		"content-details") echo 'org.sunbird.analytics.sourcing.ContentDetailsReport'
+		;;
+		*) echo $1
+		;;
+	esac		
+}
+
+if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi
+
+if [ ! -z "$1" ]; then job_config=$(config $1); else job_config="$2"; fi
+
+if [ ! -z "$2" ]; then batchIds=";$2"; else batchIds=""; fi
+
+echo "Starting the job - $1" >> "$DP_LOGS/$today-job-execution.log"
+
+echo "Job modelName - $job_id" >> "$DP_LOGS/$today-job-execution.log"
+
+nohup $SPARK_HOME/bin/spark-submit  --conf spark.driver.extraJavaOptions="-Dconfig.file=$MODELS_HOME/$file_path" --conf spark.executor.extraJavaOptions="-Dconfig.file=$MODELS_HOME/$file_path" --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.12-2.5.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config$batchIds" \ >> "$DP_LOGS/$today-job-execution.log" 2>&1
+
+echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log"
\ No newline at end of file
diff --git a/ansible/roles/data-products-deploy/templates/run-job.j2 b/ansible/roles/data-products-deploy/templates/run-job.j2
index 681d7ad6c6..26ec84da87 100644
--- a/ansible/roles/data-products-deploy/templates/run-job.j2
+++ b/ansible/roles/data-products-deploy/templates/run-job.j2
@@ -10,40 +10,74 @@ today=$(date "+%Y-%m-%d")
 
 libs_path="{{ analytics.home }}/models-{{ model_version }}/data-products-1.0"
 
-telemetry_converter_ver=0.0.8
-
 get_report_job_model_name(){
 	case "$1" in
-		"assessment-dashboard-metrics") echo 'org.sunbird.analytics.job.report.AssessmentMetricsJob'
-		;;
-		"course-dashboard-metrics") echo 'org.sunbird.analytics.job.report.CourseMetricsJob'
-		;;
 		"course-enrollment-report") echo 'org.sunbird.analytics.job.report.CourseEnrollmentJob'
 		;;
 		"course-consumption-report") echo 'org.sunbird.analytics.job.report.CourseConsumptionJob'
 		;;
+		"funnel-report") echo 'org.sunbird.analytics.sourcing.FunnelReport'
+		;;
+		"sourcing-metrics") echo 'org.sunbird.analytics.sourcing.SourcingMetrics'
+		;;
 		"admin-geo-reports") echo 'org.sunbird.analytics.job.report.StateAdminGeoReportJob'
 		;;
 		"etb-metrics") echo 'org.sunbird.analytics.job.report.ETBMetricsJob'
 		;;
 		"admin-user-reports") echo 'org.sunbird.analytics.job.report.StateAdminReportJob'
 		;;
-		"textbook-progress-report") echo 'org.sunbird.analytics.job.report.TextbookProgressJob'
-        ;;
+		"userinfo-exhaust") echo 'org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob'
+		;;
+		"response-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJob'
+		;;
+    "response-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2'
+		;;
+		"progress-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJob'
+		;;
+		"progress-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2'
+		;;
+		"cassandra-migration") echo 'org.sunbird.analytics.updater.CassandraMigratorJob'
+    ;;
+		"collection-summary-report") echo 'org.sunbird.analytics.job.report.CollectionSummaryJob'
+		;;
+		"program-collection-summary-report") echo 'org.sunbird.analytics.job.report.CollectionSummaryJob'
+		;;
+		"collection-summary-report-v2") echo 'org.sunbird.analytics.job.report.CollectionSummaryJobV2'
+		;;
+		"assessment-score-metric-correction") echo 'org.sunbird.analytics.audit.AssessmentScoreCorrectionJob'
+		;;
+		"course-batch-status-updater") echo 'org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob'
+		;;
+		"collection-reconciliation-job") echo 'org.sunbird.analytics.audit.CollectionReconciliationJob'
+		;;
+		"assessment-correction") echo 'org.sunbird.analytics.job.report.AssessmentCorrectionJob'
+	  ;;
+	  "score-metric-migration-job") echo 'org.sunbird.analytics.audit.ScoreMetricMigrationJob'
+	  ;;
+    "assessment-archival") echo 'org.sunbird.analytics.job.report.AssessmentArchivalJob'
+    ;;
+    "assessment-archived-removal") echo 'org.sunbird.analytics.job.report.AssessmentArchivalJob'
+		;;
+    "uci-private-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob'
+    ;;
+    "uci-response-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIResponseExhaustJob'
+    ;;
 		*) echo $1
 		;;
-	esac		
+	esac
 }
 
 if [ ! -z "$1" ]; then job_id=$(get_report_job_model_name $1); fi
 
-if [ -z "$2" ]; then job_config=$(config $1); else job_config="$2"; fi
+if [ ! -z "$1" ]; then job_config=$(config $1 $2); else job_config="$2"; fi
+
+if [ ! -z "$2" ]; then batchIds=";$2"; else batchIds=""; fi
 
 
 echo "Starting the job - $1" >> "$DP_LOGS/$today-job-execution.log"
 
 echo "Job modelName - $job_id" >> "$DP_LOGS/$today-job-execution.log"
 
-nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.11-2.3.2.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config" >> "$DP_LOGS/$today-job-execution.log" 2>&1
+nohup $SPARK_HOME/bin/spark-submit --master local[*] --jars $(echo ${libs_path}/lib/*.jar | tr ' ' ','),$MODELS_HOME/analytics-framework-2.0.jar,$MODELS_HOME/scruid_2.12-2.5.0.jar,$MODELS_HOME/batch-models-2.0.jar --class org.ekstep.analytics.job.JobExecutor $MODELS_HOME/batch-models-2.0.jar --model "$job_id" --config "$job_config$batchIds" >> "$DP_LOGS/$today-job-execution.log" 2>&1
 
 echo "Job execution completed - $1" >> "$DP_LOGS/$today-job-execution.log"
diff --git a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2 b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
index bce73c9339..2e613b9866 100644
--- a/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
+++ b/ansible/roles/data-products-deploy/templates/start-jobmanager.j2
@@ -9,6 +9,8 @@ export azure_storage_key={{sunbird_private_storage_account_name}}
 export azure_storage_secret={{sunbird_private_storage_account_key}}
 export reports_azure_storage_key={{sunbird_private_storage_account_name}}
 export reports_azure_storage_secret={{sunbird_private_storage_account_key}}
+export druid_storage_account_key={{sunbird_public_storage_account_name}}
+export druid_storage_account_secret={{sunbird_public_storage_account_key}}
 
 export heap_conf_str={{ spark.heap_conf_str }}
 today=$(date "+%Y-%m-%d")
@@ -16,7 +18,7 @@ today=$(date "+%Y-%m-%d")
 kill_job_manager()
 {
     echo "Killing currently running job-manager process" >> "$SERVICE_LOGS/$today-job-manager.log"
-    kill $(ps aux | grep '[j]ob-manager' | awk '{print $2}') >> "$SERVICE_LOGS/$today-job-manager.log"
+    kill $(ps aux | grep 'JobManager' | awk '{print $2}') >> "$SERVICE_LOGS/$today-job-manager.log"
 }
 
 start_job_manager()
@@ -29,7 +31,7 @@ start_job_manager()
     echo "config: $job_config" >> "$SERVICE_LOGS/$today-job-manager.log"
     nohup java $heap_conf_str -cp "$SPARK_HOME/jars/*:$MODELS_HOME/*:$MODELS_HOME/data-products-1.0/lib/*" -Dconfig.file=$MODELS_HOME/{{ env }}.conf org.ekstep.analytics.job.JobManager --config "$job_config" >> $SERVICE_LOGS/$today-job-manager.log 2>&1 &
     
-    job_manager_pid=$(ps aux | grep '[j]ob-manager' | awk '{print $2}') # Once Job is started just we are making whether job is running or not.
+    job_manager_pid=$(ps aux | grep 'JobManager' | awk '{print $2}') # Once Job is started just we are making whether job is running or not.
     if [[ ! -z "$job_manager_pid" ]]; then
         echo "Job manager is started." >> "$SERVICE_LOGS/$today-job-manager.log"
     else
diff --git a/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2 b/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2
index f16e51cc49..53c032cd29 100644
--- a/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2
+++ b/ansible/roles/data-products-deploy/templates/submit-all-jobs.rb.j2
@@ -14,8 +14,8 @@ end
 
 def submit_all_jobs
     report_jobs = { 
-    "assessment-dashboard-metrics" => "org.sunbird.analytics.job.report.AssessmentMetricsJob", 
-    "course-dashboard-metrics" => "org.sunbird.analytics.job.report.CourseMetricsJob", 
+    "assessment-dashboard-metrics" => "org.sunbird.analytics.job.report.AssessmentMetricsJobV2", 
+    "course-dashboard-metrics" => "org.sunbird.analytics.job.report.CourseMetricsJobV2", 
     "course-enrollment-report" => "org.sunbird.analytics.job.report.CourseEnrollmentJob",
     "course-consumption-report" => "org.sunbird.analytics.job.report.CourseConsumptionJob",
     "etb-metrics" => "org.sunbird.analytics.job.report.ETBMetricsJob",
diff --git a/ansible/roles/data-products-deploy/templates/submit-script.j2 b/ansible/roles/data-products-deploy/templates/submit-script.j2
new file mode 100644
index 0000000000..e8341dc1e8
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/submit-script.j2
@@ -0,0 +1,183 @@
+#!/usr/bin/env bash
+
+## Job to run daily
+cd "{{ analytics_cluster.home }}"
+source model-config.sh
+today=$(date "+%Y-%m-%d")
+
+while :; do
+    case $1 in
+        -j|--job)   shift
+                    job="$1"         
+        ;;
+        -m|--mode)  shift
+                    mode="$1"            
+        ;;
+        -p|--parallelisation)   shift
+                                parallelisation=$1            
+        ;;
+        -pa|--partitions)   shift
+                            partitions=$1       
+        ;;
+        -sd|--startDate)    shift
+                            start_date=$1            
+        ;;
+        -ed|--endDate)  shift
+                        end_date=$1            
+        ;;
+        -h|--sparkMaster)   shift
+                            sparkMaster=$1
+        ;;
+        -sp|--selectedPartitions)   shift 
+                                    selected_partitions=$1
+        ;;
+        *) break
+    esac
+    shift
+done
+
+get_report_job_model_name(){
+    case "$1" in
+        "assessment-dashboard-metrics") echo 'org.sunbird.analytics.job.report.AssessmentMetricsJobV2'
+        ;;
+        "course-dashboard-metrics") echo 'org.sunbird.analytics.job.report.CourseMetricsJobV2'
+        ;;
+        "userinfo-exhaust") echo 'org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob'
+        ;;
+        "response-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJob'
+        ;;
+        "response-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2'
+        ;;
+        "progress-exhaust") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJob'
+        ;;
+        "progress-exhaust-v2") echo 'org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2'
+        ;;
+        "cassandra-migration") echo 'org.sunbird.analytics.updater.CassandraMigratorJob'
+        ;;
+        "uci-private-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob'
+        ;;
+        "uci-response-exhaust") echo 'org.sunbird.analytics.exhaust.uci.UCIResponseExhaustJob'
+        ;;
+        *) echo $1
+        ;;
+    esac        
+}
+
+submit_cluster_job() {
+   # add batch number to config
+   echo "Running for below batch number $i"
+   batchNumberString="\\\"modelParams\\\":{\\\"batchNumber\\\":$i,"
+   job_config=$(config $job)
+   cluster_job_config=${job_config//'"'/'\"'}
+   finalConfig=${cluster_job_config/'\"modelParams\":{'/$batchNumberString}
+   echo $finalConfig
+   echo "Running $job as parallel jobs"
+   classVariable="org.ekstep.analytics.job.JobExecutor"
+   argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]"
+   argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList"
+   clusterConfig=`cat cluster-config.json`
+   requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
+   finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
+   echo $finalRequestBody
+   response=$(curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: admin_name }}")
+   echo "Submitted job for batchNumer $i below is the response"
+   echo $response
+}
+
+job_id=$(get_report_job_model_name $job)
+
+if [ -z "$sparkMaster" ]; then sparkMaster="local[*]"; else sparkMaster="$sparkMaster"; fi
+
+if [ "$mode" = "via-partition" ]; then
+    endPartitions=`expr $partitions - 1`
+    if [ -z "$parallelisation" ]; then parallelisation=1; else parallelisation=$parallelisation; fi
+    # add partitions to config and start jobs
+    for i in $(seq 0 $parallelisation $endPartitions)
+        do 
+            # add partitions to config
+            partitionString="\\\"delta\\\":0,\\\"partitions\\\":[$(seq -s , $i `expr $i + $parallelisation - 1`)]"
+            if [ -z "$start_date" ]; then
+                job_config=$(config $job)
+                cluster_job_config=${job_config//'"'/'\"'}
+                finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString}
+                echo $finalConfig
+                echo "Running $job by partitions."
+                classVariable="org.ekstep.analytics.job.JobExecutor"
+                argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]"
+            else 
+                job_config=$(config $job '__endDate__')
+                cluster_job_config=${job_config//'"'/'\"'}
+                finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString}
+                echo $finalConfig
+                echo "Running $job by partitions via Replay-Supervisor."
+                classVariable="org.ekstep.analytics.job.ReplaySupervisor"
+                argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]"
+            fi
+            argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList"
+            clusterConfig=`cat cluster-config.json`
+            requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
+            finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
+            echo $finalRequestBody
+            curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+        done
+
+elif [ "$mode" = "parallel-jobs" ]; then
+     # add batch number to config and submit jobs
+    echo "inside parallel-jobs block" 
+    echo $parallelisation
+    if [ $parallelisation -ge 1 ]; then
+        for i in $(seq 1 $parallelisation)
+            do 
+            submit_cluster_job $i &
+            done
+    else echo "No requests found in table"; fi       
+
+elif [ "$mode" = "selected-partition" ]; then
+     # add partitions to config
+     partitionString="\\\"delta\\\":0,\\\"partitions\\\":[$selected_partitions]"
+     if [ -z "$start_date" ]; then
+         job_config=$(config $job)
+         cluster_job_config=${job_config//'"'/'\"'}
+         finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString}
+         echo $finalConfig
+         echo "Running $job by partitions."
+         classVariable="org.ekstep.analytics.job.JobExecutor"
+         argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\"]"
+     else 
+         job_config=$(config $job '__endDate__')
+         cluster_job_config=${job_config//'"'/'\"'}
+         finalConfig=${cluster_job_config/'\"delta\":0'/$partitionString}
+         echo $finalConfig
+         echo "Running $job by partitions via Replay-Supervisor."
+         classVariable="org.ekstep.analytics.job.ReplaySupervisor"
+         argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$finalConfig\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]"
+     fi
+     argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList"
+     clusterConfig=`cat cluster-config.json`
+     requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
+     finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
+     echo $finalRequestBody
+     curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"
+
+else
+    if [ -z "$start_date" ]; then
+        echo "Running $job without partition via run-job."
+        job_config=$(config $job)
+        cluster_job_config=${job_config//'"'/'\"'}
+        classVariable="org.ekstep.analytics.job.JobExecutor"
+        argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$cluster_job_config\"]"
+    else   
+        job_config=$(config $job '__endDate__')
+        cluster_job_config=${job_config//'"'/'\"'}
+        echo "Running $job without partition via Replay-Supervisor." 
+        classVariable="org.ekstep.analytics.job.ReplaySupervisor"
+        argsList="\"args\": [\"--model\", \"$job_id\", \"--config\", \"$cluster_job_config\", \"--fromDate\", \"$start_date\", \"--toDate\", \"$end_date\"]"
+    fi  
+    argsStr="\"className\": \"org.ekstep.analytics.job.JobExecutor\", $argsList"
+    echo $argsStr
+    clusterConfig=`cat cluster-config.json`
+    requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr}
+    finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable}
+    echo $finalRequestBody
+    curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}"    
+fi
diff --git a/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2 b/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2
new file mode 100644
index 0000000000..cfd986b008
--- /dev/null
+++ b/ansible/roles/data-products-deploy/templates/update-job-requests.py.j2
@@ -0,0 +1,119 @@
+from __future__ import division
+import math
+import psycopg2
+import sys
+import pandas as pd
+from IPython.display import display
+from psycopg2 import sql, connect
+import json
+
+ 
+def updateExhaustRequests(db, table, update_list):
+    for r in update_list:
+        cursor = db.cursor()
+        batchNum = r['batch_number']
+        requestId = r['request_id']
+        insertQry = "UPDATE {0} SET batch_number = {1} WHERE request_id = '{2}'".format(table, batchNum, requestId)
+        n = cursor.execute(insertQry)
+
+def updateDruidRequests(db, table, update_list):
+    for r in update_list:
+        cursor = db.cursor()
+        batchNum = r['batch_number']
+        reportId = r['report_id']
+        insertQry = "UPDATE {0} SET batch_number = {1} WHERE report_id = '{2}'".format(table, batchNum, reportId)
+        n = cursor.execute(insertQry)
+
+def processRequests(totalRequestsDf, jobId, batchSize, db, table,jobType):
+    # Compute parallelism from batchSize & totalRequests
+    # update batch_number to table
+
+    totalRequests = len(totalRequestsDf.index)
+    print("totalRequests {0}".format(totalRequests))
+
+    parallelism = int(math.ceil(totalRequests/batchSize))
+    print("parallelism computed {0}".format(parallelism))
+
+    if totalRequests > 0:
+        if jobType == 'exhaust':
+          totalRequestsDf["row_num"] = totalRequestsDf.groupby(by=['job_id'])['request_id'].transform(lambda x: x.rank())
+        else:
+          totalRequestsDf["row_num"] = totalRequestsDf['report_id'].transform(lambda x: x.rank())
+        #display(totalRequestsDf)
+
+        start_index = 1
+        end_index = batchSize
+        for i in range(1, parallelism+1):
+            subSetDf = totalRequestsDf[(totalRequestsDf['row_num'] >= start_index) & (totalRequestsDf['row_num'] <= end_index)]
+            subSetDf["batch_number"] = i
+            print(start_index,end_index)
+            if jobType == 'exhaust':
+              updateExhaustRequests(db, table, json.loads(subSetDf.to_json(orient='records')))
+            else:
+              updateDruidRequests(db, table, json.loads(subSetDf.to_json(orient='records')))
+            start_index = 1 + end_index
+            end_index = end_index + batchSize
+        db.commit()
+        db.close()    
+        return parallelism
+    else:
+        return 0
+
+def postgresql_to_dataframe(db, select_query, column_names):
+    cursor = db.cursor()
+    try:
+        cursor.execute(select_query)
+    except (Exception, psycopg2.DatabaseError) as error:
+        print("Error: %s" % error)
+        return 1
+    
+    tupples = cursor.fetchall()
+    
+    df = pd.DataFrame(tupples, columns=column_names)
+    #display(df)
+    return df
+
+def get_columns_names(db,table):
+    columns = []
+    col_cursor = db.cursor()
+    col_names_str = "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS WHERE "
+    col_names_str += "table_name = '{}';".format( table )
+    try:
+        sql_object = sql.SQL(col_names_str).format(sql.Identifier( table))
+        col_cursor.execute( sql_object )
+        col_names = (col_cursor.fetchall())
+        for tup in col_names:
+           columns += [ tup[0] ]
+        col_cursor.close()
+    except Exception as err:
+        print ("get_columns_names ERROR:", err)
+    return columns
+
+def main(batchSize, jobId,jobType,table):
+    host="{{postgres.db_url}}"
+    port={{postgres.db_port}}
+    user="{{postgres.db_username}}"
+    password="{{postgres.db_password}}"
+    database="{{postgres.db_name}}"
+    url_connect = "jdbc:postgresql://{0}:{1}/{2}".format(host, port, database)
+
+    db = psycopg2.connect(host=host, user=user, password=password, database=database, port=port)
+
+    column_names = get_columns_names(db, table)
+    
+    if jobType == 'exhaust':
+      jobId = jobId.split("-v2")[0] if "-v2" in jobId else jobId
+      selectQuery = "select * from {0} where job_id = '{1}' and status IN ('SUBMITTED', 'FAILED') and iteration < 3;".format(table, jobId)
+    else:
+      selectQuery = "select * from {0} where status IN ('ACTIVE')".format(table)
+    df = postgresql_to_dataframe(db, selectQuery, column_names)
+
+    parallelism = processRequests(df, jobId, batchSize, db, table,jobType)
+    return parallelism
+
+batchSize =int(sys.argv[2])
+jobId=sys.argv[1]
+jobType = sys.argv[3]
+table = sys.argv[4]
+parallelism = main(batchSize, jobId,jobType,table)
+print("returning parallelism value: {0}".format(parallelism))
diff --git a/ansible/roles/data-products-fetch-logs/tasks/main.yml b/ansible/roles/data-products-fetch-logs/tasks/main.yml
index 7655051a8b..4c442853f5 100644
--- a/ansible/roles/data-products-fetch-logs/tasks/main.yml
+++ b/ansible/roles/data-products-fetch-logs/tasks/main.yml
@@ -53,7 +53,7 @@
 - name: Run Job
   become: yes
   become_user: "{{ analytics_user }}"
-  shell: "nohup {{ analytics.home }}/scripts/run-job.sh {{ job_id }} &"
+  shell: "nohup {{ analytics.home }}/scripts/run-job.sh {{ job_id }} {{ batch_id }} {{ keyword }} &"
   async: "{{ (pause_min * 60) }}"
   poll: 0
   tags:
diff --git a/ansible/roles/druid-anomaly-detection/defaults/main.yml b/ansible/roles/druid-anomaly-detection/defaults/main.yml
new file mode 100644
index 0000000000..9828fc2a92
--- /dev/null
+++ b/ansible/roles/druid-anomaly-detection/defaults/main.yml
@@ -0,0 +1,22 @@
+rollup_druid_broker_host: "http://{{ groups['rollup-broker'][0] | default(groups['raw-broker'][0]) }}"
+supervisord_process_name: anomaly_detection
+anomaly_detection_module_artifact: anomaly_detection.tar.gz
+anomaly_detection_module_path: /home/analytics/anomaly_detection
+virtualenv_path: "{{ anomaly_detection_module_path }}/anomaly_venv"
+anomaly_detection_job_config_file: "{{virtualenv_path}}/config.json"
+anomaly_detection_data_dir: "/home/analytics/anomaly_detection/data"
+anomaly_detection_queries:
+  content-mimetype-errors:
+    job_id: "content-mimetype-errors"
+    query_granularity: "hour"
+    query: 'WITH \"content_plays\" AS (SELECT \"content_mimetype\",\"context_pdata_id\",sum(total_count) AS \"plays\" FROM \"druid\".\"telemetry-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' AND \"edata_type\" = ''content'' AND \"edata_mode\" = ''play'' GROUP BY \"content_mimetype\",\"context_pdata_id\") SELECT cp.content_mimetype, cp.context_pdata_id, SUM(cpe.errors) * 100.0/SUM(cp.plays) AS \"perc_errors\" FROM (SELECT \"content_mimetype\", \"context_pdata_id\",SUM(\"total_count\") AS \"errors\" FROM \"druid\".\"error-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' GROUP BY \"content_mimetype\",\"context_pdata_id\") AS cpe INNER JOIN content_plays AS cp ON cp.content_mimetype = cpe.content_mimetype AND cp.context_pdata_id = cpe.context_pdata_id GROUP BY cp.content_mimetype, cp.context_pdata_id'
+    frequency: "hour"
+    cron: "0 */1 * * *"
+    offset_in_mins: 5
+  collection-type-errors:
+    job_id: "collection-type-errors"
+    query_granularity: "hour"
+    query: 'WITH \"content_plays\" AS (SELECT \"collection_type\", \"context_pdata_id\",sum(total_count) AS \"plays\" FROM \"druid\".\"telemetry-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' AND \"edata_type\" = ''content'' AND \"edata_mode\" = ''play'' GROUP BY \"collection_type\",\"context_pdata_id\") SELECT cp.collection_type,cp.context_pdata_id, SUM(cpe.errors) * 100.0/SUM(cp.plays) AS \"perc_errors\" FROM (SELECT \"collection_type\", \"context_pdata_id\",SUM(\"total_count\") AS \"errors\" FROM \"druid\".\"error-hourly-rollup-syncts\" WHERE \"__time\" BETWEEN TIMESTAMP ''{start_time}'' AND TIMESTAMP ''{end_time}'' GROUP BY \"collection_type\",\"context_pdata_id\") AS cpe INNER JOIN content_plays AS cp ON cp.collection_type = cpe.collection_type AND cp.context_pdata_id = cpe.context_pdata_id GROUP BY cp.collection_type, cp.context_pdata_id'
+    frequency: "hour"
+    cron: "0 */1 * * *"
+    offset_in_mins: 5
\ No newline at end of file
diff --git a/ansible/roles/druid-anomaly-detection/tasks/main.yml b/ansible/roles/druid-anomaly-detection/tasks/main.yml
new file mode 100644
index 0000000000..91e9b2e6ae
--- /dev/null
+++ b/ansible/roles/druid-anomaly-detection/tasks/main.yml
@@ -0,0 +1,67 @@
+---
+
+- name: Install supervisord
+  apt:
+    name: supervisor
+    state: present
+    update_cache: yes
+  become: true
+
+- name: copy supervisor conf
+  template: src="supervisor.conf.j2" dest="/etc/supervisor/conf.d/anomaly_detection.conf"
+  become: true
+
+- name: Start supervisord
+  service:
+    name: "supervisor"
+    state: started
+    enabled: yes
+  become: true
+
+- name: Add deadsnakes repo
+  apt_repository: repo="ppa:deadsnakes/ppa"
+  become: true
+
+- name: Install python
+  apt: 
+    name:
+      - python3.8-dev
+      - python3-pip
+      - python3-setuptools
+      - python3.8-venv
+    state: present
+  become: true
+
+- name: create module directories
+  file:
+    path: "{{ item }}"
+    state: directory
+    owner: "{{ analytics_user }}"
+    group: "{{ analytics_user }}"
+    mode: 0775
+  loop:
+    - "{{ anomaly_detection_module_path }}/anomaly_venv"
+    - "{{ anomaly_detection_module_path }}/data"
+  become_user: "{{ analytics_user }}"
+
+- name: Copy the anomaly detection artifact
+  copy: src={{ anomaly_detection_module_artifact }} dest={{ anomaly_detection_module_path }}
+  become_user: "{{ analytics_user }}"
+
+- name: Installing anomaly detection as a python module
+  pip:
+    name: "{{ anomaly_detection_module_path }}/{{ anomaly_detection_module_artifact }}"
+    virtualenv: "{{ virtualenv_path }}"
+    virtualenv_command: "/usr/bin/python3.8 -m venv"
+  become_user: "{{ analytics_user }}"
+
+- name: copy query config to config.json
+  template: src="config.json.j2" dest="{{ virtualenv_path }}/config.json"
+  with_items: "{{ anomaly_detection_queries | list }}"
+  become_user: "{{ analytics_user }}"
+
+- name: Start anomaly detection process
+  supervisorctl:
+    name: "anomaly_detection"
+    state: restarted
+  become: true
\ No newline at end of file
diff --git a/ansible/roles/druid-anomaly-detection/templates/config.json.j2 b/ansible/roles/druid-anomaly-detection/templates/config.json.j2
new file mode 100644
index 0000000000..0faf6c00ad
--- /dev/null
+++ b/ansible/roles/druid-anomaly-detection/templates/config.json.j2
@@ -0,0 +1,14 @@
+{
+  "jobs": [
+  	{% for item in anomaly_detection_queries.values() %}
+    {
+      "job_id": "{{ item.job_id }}",
+      "query_granularity": "{{ item.query_granularity }}",
+      "query": "{{ item.query }}",
+      "cron": "{{ item.cron }}",
+      "offset_in_mins": {{ item.offset_in_mins }},
+      "frequency": "{{ item.frequency }}"
+    }{{ "," if not loop.last else "" }}
+    {% endfor %}
+  ]
+}
\ No newline at end of file
diff --git a/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2 b/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2
new file mode 100644
index 0000000000..2541c43e82
--- /dev/null
+++ b/ansible/roles/druid-anomaly-detection/templates/supervisor.conf.j2
@@ -0,0 +1,10 @@
+[program:{{ supervisord_process_name }}]
+process_name={{ supervisord_process_name }}
+command={{ virtualenv_path }}/bin/anomaly_detection druid_anomaly_detection --config_file={{ anomaly_detection_job_config_file }} --data_dir="{{ anomaly_detection_data_dir }}" --druid_broker_host="{{ rollup_druid_broker_host }}" --druid_broker_port=8082
+environment=PATH="{{ virtualenv_path }}/bin:%(ENV_PATH)s"
+user={{analytics_user}}
+group={{analytics_user}}
+killasgroup=true
+startsecs=5
+stopwaitsecs=5
+redirect_stderr=True
diff --git a/ansible/roles/druid-ingestion/defaults/main.yml b/ansible/roles/druid-ingestion/defaults/main.yml
index 2960c95a7b..2261c4389d 100644
--- a/ansible/roles/druid-ingestion/defaults/main.yml
+++ b/ansible/roles/druid-ingestion/defaults/main.yml
@@ -1,17 +1,41 @@
 remote_user: druid
 broker_port: 8082
 druid_home: /home/druid
-druid_schema:
- - telemetry_index_kafka
- - summary_index_kafka
- - log_index_kafka
- - telemetry_feedback_index_kafka
- - error_index_kafka
- - pipeline_metrics_index_kafka
-event_types:
- - telemetry-events
- - summary-events
- - telemetry-log-events
- - telemetry-feedback-events
- - telemetry-error-events
- - pipeline-metrics
+desktop_pdata_id: "{{env}}.sunbird.desktop"
+
+druid_mm_heap_dump_file: /var/log/druid/crash_logs/middlemanager.hprof
+druid_overlord_port: 8090
+
+ingestion_spec_configs:
+  raw_telemetry_events:
+    minTaskMemory: 200m
+    maxTaskMemory: 400m
+
+raw_offline_desktop_taskcount: 1
+raw_summary_events_taskcount: 1
+raw_telemetry_events_taskcount: 1
+raw_telemetry_feedback_events_taskcount: 1
+rollup_error_hourly_syncts_taskcount: 1
+rollup_summary_distinct_counts_taskcount: 1
+rollup_summary_syncts_taskcount: 1
+rollup_telemetry_audit_syncts_taskcount: 1
+rollup_telemetry_hourly_syncts_taskcount: 1
+rollup_telemetry_syncts_taskcount: 1
+rollup_tpd_hourly_taskcount: 1
+ml_observation_task_count: 1
+ml_observation_evidence_task_count: 1
+ml_survey_task_count: 1
+ml_survey_evidence_task_count: 1
+rollup_ml_project_taskcount: 1
+rollup_course_completion_audit_syncts_taskcount: 1
+container_name: "telemetry-data-store"
+sl_survey_evidence_task_count: 1
+sl_observation_task_count: 1
+sl_observation_evidence_task_count: 1
+sl_survey_task_count: 1
+rollup_course_completion_audit_syncts_taskcount: 1
+rollup_ml_project_taskcount: 1
+rollup_ml_observation_status_taskcount: 1
+rollup_ml_project_status_taskcount: 1
+rollup_ml_survey_status_taskcount: 1
+
diff --git a/ansible/roles/druid-ingestion/tasks/main.yml b/ansible/roles/druid-ingestion/tasks/main.yml
index 199345fd4e..da8e8c5899 100644
--- a/ansible/roles/druid-ingestion/tasks/main.yml
+++ b/ansible/roles/druid-ingestion/tasks/main.yml
@@ -1,21 +1,35 @@
 
-- name: Copy the schema file
-  template: src="{{item}}" dest="{{druid_home}}/{{item}}.json"
-  with_items: "{{druid_schema}}"
+- name: Load the context json
+  set_fact: {"{{ item }}_context":"{{ lookup('template', 'context.json.j2')}}"}
+  with_items: "{{ ingestion_spec_configs }}"
   tags: start-task
 
+- name: Load the ingestion json
+  set_fact: {"{{ item }}_json":"{{ lookup('template', (item))}}"}
+  with_items: "{{ ingestion_task_names.split(',')|list }}"
+  tags: start-task
+
+- name: Override the schema spec with context and  Copy schema file
+  copy:
+     content: "{{ (lookup('vars', item + '_context' , default={}) | combine(vars[(item + '_json')])) | to_json(indent=4) }}"
+     dest: "{{druid_home}}/{{item}}.json"
+  become_user: "{{remote_user}}"
+  with_items: "{{ ingestion_task_names.split(',')|list }}"
+  tags: start-task
+
+
 - name: Start kafka supervisor tasks for defined datasources
-  shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{overlord_host}}:8090/druid/indexer/v1/supervisor'
-  with_items: "{{druid_schema}}"
+  shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor'
+  with_items: "{{ ingestion_task_names.split(',')|list }}"
   become_user: "{{remote_user}}" 
   tags: start-task
 
 - name: Stop kafka supervisor tasks for defined datasources
-  shell: 'curl -X POST http://{{overlord_host}}:8090/druid/indexer/v1/supervisor/{{item}}/terminate'
-  with_items: "{{event_types}}"
+  shell: 'curl -X POST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/supervisor/{{item}}/terminate'
+  with_items: "{{ ingestion_task_names.split(',')|list }}"
   tags: stop-task
 
 - name: Stop currently running ingestion tasks
-  shell: 'curl -XPOST http://{{overlord_host}}:8090/druid/indexer/v1/datasources/{{item}}/shutdownAllTasks'
-  with_items: "{{event_types}}"
+  shell: 'curl -XPOST http://{{overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/datasources/{{item}}/shutdownAllTasks'
+  with_items: "{{ ingestion_task_names.split(',')|list }}"
   tags: stop-task
diff --git a/ansible/roles/druid-ingestion/templates/context.json.j2 b/ansible/roles/druid-ingestion/templates/context.json.j2
new file mode 100644
index 0000000000..67be15a335
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/context.json.j2
@@ -0,0 +1 @@
+{"context":{"druid.indexer.runner.javaOpts":"-server -Xms{{ ingestion_spec_configs[item].minTaskMemory }} -Xmx{{ ingestion_spec_configs[item].maxTaskMemory }} -XX:+UseG1GC -XX:HeapDumpPath=/var/log/druid/crash_logs/middlemanager.hprof -XX:MaxGCPauseMillis=100"}}
diff --git a/ansible/roles/druid-ingestion/templates/error_index_kafka b/ansible/roles/druid-ingestion/templates/error_index_kafka
deleted file mode 100644
index cef137dc42..0000000000
--- a/ansible/roles/druid-ingestion/templates/error_index_kafka
+++ /dev/null
@@ -1,420 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "telemetry-error-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "ets"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "ver"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_err",
-              "expr": "$.edata.err"
-            },
-            {
-              "type": "path",
-              "name": "edata_errtype",
-              "expr": "$.edata.errtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_pageid",
-              "expr": "$.edata.pageid"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_id",
-              "expr": "$.edata.object.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_type",
-              "expr": "$.edata.object.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_ver",
-              "expr": "$.edata.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_subtype",
-              "expr": "$.edata.object.subtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_name",
-              "expr": "$.edata.object.name"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_code",
-              "expr": "$.edata.object.code"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_parent_id",
-              "expr": "$.edata.object.parent.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_object_parent_type",
-              "expr": "$.edata.object.parent.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_id",
-              "expr": "$.edata.plugin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_ver",
-              "expr": "$.edata.plugin.ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_category",
-              "expr": "$.edata.plugin.category"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "long",
-              "name": "ets"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "string",
-              "name": "ver"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "type": "string",
-              "name": "context_cdata_type"
-            },
-            {
-              "type": "string",
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "tags"
-            },
-            {
-              "type": "string",
-              "name": "edata_err"
-            },
-            {
-              "type": "string",
-              "name": "edata_errtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_pageid"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_subtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_name"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_code"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_parent_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_object_parent_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_category"
-            }
-          ]
-        },
-        "timestampSpec": {
-          "column": "syncts",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.error",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false,
-    "logParseExceptions": true,
-    "maxSavedParseExceptions": 10
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/log_index_kafka b/ansible/roles/druid-ingestion/templates/log_index_kafka
deleted file mode 100644
index f8cbfdb066..0000000000
--- a/ansible/roles/druid-ingestion/templates/log_index_kafka
+++ /dev/null
@@ -1,465 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "telemetry-log-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "syncts"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "ver"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_type",
-              "expr": "$.edata.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_level",
-              "expr": "$.edata.level"
-            },
-            {
-              "type": "path",
-              "name": "edata_message",
-              "expr": "$.edata.message"
-            },
-            {
-              "type": "path",
-              "name": "edata_pageid",
-              "expr": "$.edata.pageid"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_rid",
-              "expr": "$.edata.params[*].rid"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_uip",
-              "expr": "$.edata.params[*].uip"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_title",
-              "expr": "$.edata.params[*].title"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_category",
-              "expr": "$.edata.params[*].category"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_url",
-              "expr": "$.edata.params[*].url"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_size",
-              "expr": "$.edata.params[*].size"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_duration",
-              "expr": "$.edata.params[*].duration"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_status",
-              "expr": "$.edata.params[*].status"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_protocol",
-              "expr": "$.edata.params[*].protocol"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_method",
-              "expr": "$.edata.params[*].method"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_req",
-              "expr": "$.edata.params[*].req"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_consumer_id",
-              "expr": "$.edata.params[*].consumer_id"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_ver",
-              "expr": "$.edata.params[*].ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_events_count",
-              "expr": "$.edata.params[*].events_count"
-            },
-            {
-              "type": "path",
-              "name": "edata_params_",
-              "expr": "$.edata.params[*].sync_status"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "long",
-              "name": "syncts"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "string",
-              "name": "ver"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "type": "string",
-              "name": "context_cdata_type"
-            },
-            {
-              "type": "string",
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "tags"
-            },
-            {
-              "type": "string",
-              "name": "edata_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_level"
-            },
-            {
-              "type": "string",
-              "name": "edata_message"
-            },
-            {
-              "type": "string",
-              "name": "edata_pageid"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_rid"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_uip"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_title"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_category"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_url"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_size"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_duration"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_status"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_protocol"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_method"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_req"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_consumer_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_events_count"
-            },
-            {
-              "type": "string",
-              "name": "edata_params_events_sync_status"
-            }
-          ]
-        },
-        "timestampSpec": {
-          "column": "ets",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.log",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false,
-    "logParseExceptions": true,
-    "maxSavedParseExceptions": 10
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/offline_desktop_kafka b/ansible/roles/druid-ingestion/templates/offline_desktop_kafka
deleted file mode 100644
index b1b050facb..0000000000
--- a/ansible/roles/druid-ingestion/templates/offline_desktop_kafka
+++ /dev/null
@@ -1,1353 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "offline-desktop-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "syncts"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_type",
-              "expr": "$.edata.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_subtype",
-              "expr": "$.edata.subtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_mode",
-              "expr": "$.edata.mode"
-            },
-            {
-              "type": "path",
-              "name": "edata_pageid",
-              "expr": "$.edata.pageid"
-            },
-            {
-              "type": "path",
-              "name": "edata_uri",
-              "expr": "$.edata.uri"
-            },
-            {
-              "type": "path",
-              "name": "edata_id",
-              "expr": "$.edata.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_duration",
-              "expr": "$.edata.duration"
-            },
-            {
-              "type": "path",
-              "name": "edata_index",
-              "expr": "$.edata.index"
-            },
-            {
-              "type": "path",
-              "name": "edata_pass",
-              "expr": "$.edata.pass"
-            },
-            {
-              "type": "path",
-              "name": "edata_score",
-              "expr": "$.edata.score"
-            },
-            {
-              "type": "path",
-              "name": "edata_resvalues",
-              "expr": "$.edata.resvalues[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_id",
-              "expr": "$.edata.item.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_title",
-              "expr": "$.edata.item.title"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_maxscore",
-              "expr": "$.edata.item.maxscore"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_id",
-              "expr": "$.edata.target.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_type",
-              "expr": "$.edata.target.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_rating",
-              "expr": "$.edata.rating"
-            },
-            {
-              "type": "path",
-              "name": "edata_comments",
-              "expr": "$.edata.comments"
-            },
-            {
-              "type": "path",
-              "name": "edata_dir",
-              "expr": "$.edata.dir"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_id",
-              "expr": "$.edata.items[*].id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_type",
-              "expr": "$.edata.items[*].type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_id",
-              "expr": "$.edata.items[*].origin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_type",
-              "expr": "$.edata.items[*].origin.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_id",
-              "expr": "$.edata.items[*].to.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_type",
-              "expr": "$.edata.items[*].to.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_id",
-              "expr": "$.edata.plugin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_ver",
-              "expr": "$.edata.plugin.ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_category",
-              "expr": "$.edata.plugin.category"
-            },
-            {
-              "type": "path",
-              "name": "edata_props",
-              "expr": "$.edata.props[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_state",
-              "expr": "$.edata.state"
-            },
-            {
-              "type": "path",
-              "name": "edata_prevstate",
-              "expr": "$.edata.prevstate"
-            },
-            {
-              "type": "path",
-              "name": "edata_size",
-              "expr": "$.edata.size"
-            },
-            {
-              "type": "path",
-              "name": "edata_filters_dialcodes",
-              "expr": "$.edata.filters.dialcodes"
-            },
-            {
-              "type": "path",
-              "name": "edata_topn_identifier",
-              "expr": "$.edata.topn[*].identifier"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objid",
-              "expr": "$.edata.visits[*].objid"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objtype",
-              "expr": "$.edata.visits[*].objtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objver",
-              "expr": "$.edata.visits[*].objver"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_index",
-              "expr": "$.edata.visits[*].index"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state",
-              "expr": "$.devicedata.state"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_code",
-              "expr": "$.devicedata.statecode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_city",
-              "expr": "$.devicedata.city"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country_code",
-              "expr": "$.devicedata.countrycode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country",
-              "expr": "$.devicedata.country"
-            },
-            {
-              "type": "path",
-              "name": "device_os",
-              "expr": "$.devicedata.devicespec.os"
-            },
-            {
-              "type": "path",
-              "name": "device_make",
-              "expr": "$.devicedata.devicespec.make"
-            },
-            {
-              "type": "path",
-              "name": "device_id",
-              "expr": "$.devicedata.devicespec.id"
-            },
-            {
-              "type": "path",
-              "name": "device_mem",
-              "expr": "$.devicedata.devicespec.mem"
-            },
-            {
-              "type": "path",
-              "name": "device_idisk",
-              "expr": "$.devicedata.devicespec.idisk"
-            },
-            {
-              "type": "path",
-              "name": "device_edisk",
-              "expr": "$.devicedata.devicespec.edisk"
-            },
-            {
-              "type": "path",
-              "name": "device_scrn",
-              "expr": "$.devicedata.devicespec.scrn"
-            },
-            {
-              "type": "path",
-              "name": "device_camera",
-              "expr": "$.devicedata.devicespec.camera"
-            },
-            {
-              "type": "path",
-              "name": "device_cpu",
-              "expr": "$.devicedata.devicespec.cpu"
-            },
-            {
-              "type": "path",
-              "name": "device_sims",
-              "expr": "$.devicedata.devicespec.sims"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_agent",
-              "expr": "$.devicedata.uaspec.agent"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_ver",
-              "expr": "$.devicedata.uaspec.ver"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_system",
-              "expr": "$.devicedata.uaspec.system"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_platform",
-              "expr": "$.devicedata.uaspec.platform"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_raw",
-              "expr": "$.devicedata.uaspec.raw"
-            },
-            {
-              "type": "path",
-              "name": "device_first_access",
-              "expr": "$.devicedata.firstaccess"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_code",
-              "expr": "$.devicedata.statecustomcode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_name",
-              "expr": "$.devicedata.statecustomname"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_district",
-              "expr": "$.devicedata.districtcustom"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_state",
-              "expr": "$.devicedata.userdeclared.state"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_district",
-              "expr": "$.devicedata.userdeclared.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_state",
-              "expr": "$.derivedlocationdata.state"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_district",
-              "expr": "$.derivedlocationdata.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_from",
-              "expr": "$.derivedlocationdata.from"
-            },
-            {
-              "type": "path",
-              "name": "content_name",
-              "expr": "$.contentdata.name"
-            },
-            {
-              "type": "path",
-              "name": "content_object_type",
-              "expr": "$.contentdata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "content_type",
-              "expr": "$.contentdata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "content_media_type",
-              "expr": "$.contentdata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "content_language",
-              "expr": "$.contentdata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_medium",
-              "expr": "$.contentdata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_gradelevel",
-              "expr": "$.contentdata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_subjects",
-              "expr": "$.contentdata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_mimetype",
-              "expr": "$.contentdata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "content_framework",
-              "expr": "$.contentdata.framework"
-            },
-            {
-              "type": "path",
-              "name": "content_board",
-              "expr": "$.contentdata.board"
-            },
-            {
-              "type": "path",
-              "name": "content_status",
-              "expr": "$.contentdata.status"
-            },
-            {
-              "type": "path",
-              "name": "content_version",
-              "expr": "$.contentdata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "content_last_submitted_on",
-              "expr": "$.contentdata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_published_on",
-              "expr": "$.contentdata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_updated_on",
-              "expr": "$.contentdata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "content_created_by",
-              "expr": "$.contentdata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "content_created_for",
-              "expr": "$.contentdata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "collection_name",
-              "expr": "$.collectiondata.name"
-            },
-            {
-              "type": "path",
-              "name": "collection_object_type",
-              "expr": "$.collectiondata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_type",
-              "expr": "$.collectiondata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_media_type",
-              "expr": "$.collectiondata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "collection_language",
-              "expr": "$.collectiondata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_medium",
-              "expr": "$.collectiondata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_gradelevel",
-              "expr": "$.collectiondata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_subjects",
-              "expr": "$.collectiondata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_mimetype",
-              "expr": "$.collectiondata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "collection_framework",
-              "expr": "$.collectiondata.framework"
-            },
-            {
-              "type": "path",
-              "name": "collection_board",
-              "expr": "$.collectiondata.board"
-            },
-            {
-              "type": "path",
-              "name": "collection_status",
-              "expr": "$.collectiondata.status"
-            },
-            {
-              "type": "path",
-              "name": "collection_version",
-              "expr": "$.collectiondata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_submitted_on",
-              "expr": "$.collectiondata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_published_on",
-              "expr": "$.collectiondata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_updated_on",
-              "expr": "$.collectiondata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_by",
-              "expr": "$.collectiondata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_for",
-              "expr": "$.collectiondata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "user_grade_list",
-              "expr": "$.userdata.gradelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_language_list",
-              "expr": "$.userdata.languagelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_subject_list",
-              "expr": "$.userdata.subjectlist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_type",
-              "expr": "$.userdata.usertype"
-            },
-            {
-              "type": "path",
-              "name": "user_roles",
-              "expr": "$.userdata.roles[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_state",
-              "expr": "$.userdata.state"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_district",
-              "expr": "$.userdata.district"
-            },
-            {
-              "type": "path",
-              "name": "user_signin_type",
-              "expr": "$.userdata.usersignintype"
-            },
-            {
-              "type": "path",
-              "name": "user_login_type",
-              "expr": "$.userdata.userlogintype"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_channel",
-              "expr": "$.dialcodedata.channel"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_batchcode",
-              "expr": "$.dialcodedata.batchcode"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_publisher",
-              "expr": "$.dialcodedata.publisher"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_generated_on",
-              "expr": "$.dialcodedata.generatedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_published_on",
-              "expr": "$.dialcodedata.publishedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_status",
-              "expr": "$.dialcodedata.status"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_object_type",
-              "expr": "$.dialcodedata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_iso_state_code",
-              "expr": "$.devicedata.iso3166statecode"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "long",
-              "name": "syncts"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "name": "context_cdata_type"
-            },
-            {
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "name": "tags"
-            },
-            {
-              "type": "string",
-              "name": "edata_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_subtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_mode"
-            },
-            {
-              "type": "string",
-              "name": "edata_pageid"
-            },
-            {
-              "type": "string",
-              "name": "edata_uri"
-            },
-            {
-              "type": "string",
-              "name": "edata_id"
-            },
-            {
-              "type": "double",
-              "name": "edata_duration"
-            },
-            {
-              "type": "long",
-              "name": "edata_index"
-            },
-            {
-              "type": "string",
-              "name": "edata_pass"
-            },
-            {
-              "type": "double",
-              "name": "edata_score"
-            },
-            {
-              "name": "edata_resvalues"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_title"
-            },
-            {
-              "type": "double",
-              "name": "edata_item_maxscore"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_type"
-            },
-            {
-              "type": "long",
-              "name": "edata_rating"
-            },
-            {
-              "type": "string",
-              "name": "edata_comments"
-            },
-            {
-              "type": "string",
-              "name": "edata_dir"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_category"
-            },
-            {
-              "name": "edata_props"
-            },
-            {
-              "type": "string",
-              "name": "edata_state"
-            },
-            {
-              "type": "string",
-              "name": "edata_prevstate"
-            },
-            {
-              "type": "long",
-              "name": "edata_size"
-            },
-            {
-              "name": "edata_filters_dialcodes"
-            },
-            {
-              "type": "string",
-              "name": "edata_topn_identifier"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objid"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objver"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_index"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_city"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country"
-            },
-            {
-              "type": "string",
-              "name": "device_os"
-            },
-            {
-              "type": "string",
-              "name": "device_make"
-            },
-            {
-              "type": "string",
-              "name": "device_id"
-            },
-            {
-              "type": "long",
-              "name": "device_mem"
-            },
-            {
-              "type": "string",
-              "name": "device_idisk"
-            },
-            {
-              "type": "string",
-              "name": "device_edisk"
-            },
-            {
-              "type": "string",
-              "name": "device_scrn"
-            },
-            {
-              "type": "string",
-              "name": "device_camera"
-            },
-            {
-              "type": "string",
-              "name": "device_cpu"
-            },
-            {
-              "type": "long",
-              "name": "device_sims"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_agent"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_ver"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_system"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_platform"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_raw"
-            },
-            {
-              "type": "long",
-              "name": "device_first_access"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_name"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_state"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_from"
-            },
-            {
-              "type": "string",
-              "name": "content_name"
-            },
-            {
-              "type": "string",
-              "name": "content_object_type"
-            },
-            {
-              "type": "string",
-              "name": "content_type"
-            },
-            {
-              "type": "string",
-              "name": "content_media_type"
-            },
-            {
-              "name": "content_language"
-            },
-            {
-              "name": "content_medium"
-            },
-            {
-              "name": "content_gradelevel"
-            },
-            {
-              "name": "content_subjects"
-            },
-            {
-              "type": "string",
-              "name": "content_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "content_framework"
-            },
-            {
-              "type": "string",
-              "name": "content_board"
-            },
-            {
-              "type": "string",
-              "name": "content_status"
-            },
-            {
-              "type": "double",
-              "name": "content_version"
-            },
-            {
-              "type": "long",
-              "name": "content_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "content_created_by"
-            },
-            {
-              "name": "content_created_for"
-            },
-            {
-              "type": "string",
-              "name": "collection_name"
-            },
-            {
-              "type": "string",
-              "name": "collection_object_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_media_type"
-            },
-            {
-              "name": "collection_language"
-            },
-            {
-              "name": "collection_medium"
-            },
-            {
-              "name": "collection_gradelevel"
-            },
-            {
-              "name": "collection_subjects"
-            },
-            {
-              "type": "string",
-              "name": "collection_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "collection_framework"
-            },
-            {
-              "type": "string",
-              "name": "collection_board"
-            },
-            {
-              "type": "string",
-              "name": "collection_status"
-            },
-            {
-              "type": "double",
-              "name": "collection_version"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "collection_created_by"
-            },
-            {
-              "name": "collection_created_for"
-            },
-            {
-              "name": "user_grade_list"
-            },
-            {
-              "name": "user_language_list"
-            },
-            {
-              "name": "user_subject_list"
-            },
-            {
-              "type": "string",
-              "name": "user_type"
-            },
-            {
-              "name": "user_roles"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_signin_type"
-            },
-            {
-              "type": "string",
-              "name": "user_login_type"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_channel"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_batchcode"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_publisher"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_generated_on"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_published_on"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_status"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_object_type"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_iso_state_code"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "ets",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    },
-    "transformSpec": {
-      "filter": {
-        "type": "selector",
-        "dimension": "context_pdata_id",
-        "value": "prod.diksha.desktop"
-      }
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.telemetry",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false,
-    "completionTimeout": "PT30M"
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka b/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka
deleted file mode 100644
index 66089429fc..0000000000
--- a/ansible/roles/druid-ingestion/templates/pipeline_metrics_index_kafka
+++ /dev/null
@@ -1,391 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "pipeline-metrics",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "job-name"
-            },
-            {
-              "type": "root",
-              "name": "success-message-count"
-            },
-            {
-              "type": "root",
-              "name": "failed-message-count"
-            },
-            {
-              "type": "root",
-              "name": "error-message-count"
-            },
-            {
-              "type": "root",
-              "name": "skipped-message-count"
-            },
-            {
-              "type": "root",
-              "name": "batch-success-count"
-            },
-            {
-              "type": "root",
-              "name": "batch-error-count"
-            },
-            {
-              "type": "root",
-              "name": "primary-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "secondary-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "consumer-lag"
-            },
-            {
-              "type": "root",
-              "name": "partition"
-            },
-            {
-              "type": "root",
-              "name": "cache-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "cache-miss-count"
-            },
-            {
-              "type": "root",
-              "name": "cache-expired-count"
-            },
-            {
-              "type": "root",
-              "name": "cache-error-count"
-            },
-            {
-              "type": "root",
-              "name": "cache-empty-values-count"
-            },
-            {
-              "type": "root",
-              "name": "processed-message-count"
-            },
-            {
-              "type": "root",
-              "name": "unprocessed-message-count"
-            },
-            {
-              "type": "root",
-              "name": "db-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "db-error-count"
-            },
-            {
-              "type": "root",
-              "name": "device-cache-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "user-cache-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "device-db-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "user-db-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "device-db-error-count"
-            },
-            {
-              "type": "root",
-              "name": "user-db-error-count"
-            },
-            {
-              "type": "root",
-              "name": "expired-event-count"
-            },
-            {
-              "type": "root",
-              "name": "duplicate-event-count"
-            },
-            {
-              "type": "root",
-              "name": "device-db-update-count"
-            },
-            {
-              "type": "root",
-              "name": "device-cache-update-count"
-            },
-            {
-              "type": "root",
-              "name": "user-declared-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "ip-location-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "no-cache-hit-count"
-            },
-            {
-              "type": "root",
-              "name": "audit-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "db-insert-count"
-            },
-            {
-              "type": "root",
-              "name": "db-update-count"
-            },
-            {
-              "type": "root",
-              "name": "share-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "dialcodes-count"
-            },
-            {
-              "type": "root",
-              "name": "dialcodes-api-hit"
-            },
-            {
-              "type": "root",
-              "name": "dialcodes-cache-hit"
-            },
-            {
-              "type": "root",
-              "name": "assess-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "log-route-success-count"
-            },
-            {
-              "type": "root",
-              "name": "error-route-success-count"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "job-name"
-            },
-            {
-              "type": "long",
-              "name": "success-message-count"
-            },
-            {
-              "type": "long",
-              "name": "failed-message-count"
-            },
-            {
-              "type": "long",
-              "name": "error-message-count"
-            },
-            {
-              "type": "long",
-              "name": "batch-success-count"
-            },
-            {
-              "type": "long",
-              "name": "batch-error-count"
-            },
-            {
-              "type": "long",
-              "name": "primary-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "secondary-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "skipped-message-count"
-            },
-            {
-              "type": "long",
-              "name": "consumer-lag"
-            },
-            {
-              "type": "long",
-              "name": "partition"
-            },
-            {
-              "type": "long",
-              "name": "cache-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "cache-miss-count"
-            },
-            {
-              "type": "long",
-              "name": "cache-expired-count"
-            },
-            {
-              "type": "long",
-              "name": "cache-error-count"
-            },
-            {
-              "type": "long",
-              "name": "cache-empty-values-count"
-            },
-            {
-              "type": "long",
-              "name": "processed-message-count"
-            },
-            {
-              "type": "long",
-              "name": "unprocessed-message-count"
-            },
-            {
-              "type": "long",
-              "name": "db-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "db-error-count"
-            },
-            {
-              "type": "long",
-              "name": "device-cache-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "user-cache-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "device-db-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "user-db-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "device-db-error-count"
-            },
-            {
-              "type": "long",
-              "name": "user-db-error-count"
-            },
-            {
-              "type": "long",
-              "name": "expired-event-count"
-            },
-            {
-              "type": "long",
-              "name": "duplicate-event-count"
-            },
-            {
-              "type": "long",
-              "name": "device-db-update-count"
-            },
-            {
-              "type": "long",
-              "name": "device-cache-update-count"
-            },
-            {
-              "type": "long",
-              "name": "user-declared-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "ip-location-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "no-cache-hit-count"
-            },
-            {
-              "type": "long",
-              "name": "audit-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "db-insert-count"
-            },
-            {
-              "type": "long",
-              "name": "db-update-count"
-            },
-            {
-              "type": "long",
-              "name": "share-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "dialcodes-count"
-            },
-            {
-              "type": "long",
-              "name": "dialcodes-api-hit"
-            },
-            {
-              "type": "long",
-              "name": "dialcodes-cache-hit"
-            },
-            {
-              "type": "long",
-              "name": "assess-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "log-route-success-count"
-            },
-            {
-              "type": "long",
-              "name": "error-route-success-count"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "metricts",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.pipeline_metrics",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index
new file mode 100644
index 0000000000..fa3558a22e
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_criteria_index
@@ -0,0 +1,137 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/observation/domain_criteria/ml_observation_domain_criteria.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "ml-obs-domain-criteria",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "time_stamp",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "solution_name"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "string",
+            "name": "solution_type"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "domain_name"
+          },
+          {
+            "type": "string",
+            "name": "domain_externalId"
+          },
+          {
+            "type": "string",
+            "name": "domain_level"
+          },
+          {
+            "type": "string",
+            "name": "criteria_name"
+          },
+          {
+            "type": "string",
+            "name": "criteria_score"
+          },
+          {
+            "type": "string",
+            "name": "criteria_id"
+          },
+          {
+            "type": "long",
+            "name": "unique_submissions"
+          },
+          {
+            "type": "long",
+            "name": "unique_entities"
+          },
+          {
+            "type": "long",
+            "name": "unique_users"
+          },
+          {
+            "type": "string",
+            "name": "time_stamp"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index
new file mode 100644
index 0000000000..37eade976e
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_domain_index
@@ -0,0 +1,125 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/observation/domain/ml_observation_domain.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "ml-obs-domain",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "time_stamp",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "solution_name"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "string",
+            "name": "solution_type"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "domain_name"
+          },
+          {
+            "type": "string",
+            "name": "domain_externalId"
+          },
+          {
+            "type": "string",
+            "name": "domain_level"
+          },
+          {
+            "type": "long",
+            "name": "unique_submissions"
+          },
+          {
+            "type": "long",
+            "name": "unique_entities"
+          },
+          {
+            "type": "long",
+            "name": "unique_users"
+          },
+          {
+            "type": "string",
+            "name": "time_stamp"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index b/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index
new file mode 100644
index 0000000000..84f15cf77a
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_ml_obs_status_index
@@ -0,0 +1,117 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/observation/ml_observation_status.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "ml-obs-status",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "time_stamp",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "solution_name"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "status"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "string",
+            "name": "solution_type"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "long",
+            "name": "unique_submissions"
+          },
+          {
+            "type": "long",
+            "name": "unique_entities"
+          },
+          {
+            "type": "long",
+            "name": "unique_users"
+          },
+          {
+            "type": "string",
+            "name": "time_stamp"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index b/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index
new file mode 100644
index 0000000000..bb214cf72b
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_ml_project_program_level_index
@@ -0,0 +1,93 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/projects/program_level/ml_projects_programLevel.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "ml-project-programLevel-status",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "time_stamp",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "status_of_project"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "project_created_type"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "long",
+            "name": "unique_projects"
+          },
+          {
+            "type": "long",
+            "name": "unique_users"
+          },
+          {
+            "type": "long",
+            "name": "no_of_imp_with_evidence"
+          },
+          {
+            "type": "string",
+            "name": "time_stamp"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index
new file mode 100644
index 0000000000..99f875c63c
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_ml_project_status_index
@@ -0,0 +1,117 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/projects/ml_projects.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "ml-project-status",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "time_stamp",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "project_title"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "status_of_project"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "project_created_type"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "long",
+            "name": "unique_projects"
+          },
+          {
+            "type": "long",
+            "name": "unique_users"
+          },
+          {
+            "type": "long",
+            "name": "no_of_imp_with_evidence"
+          },
+          {
+            "type": "string",
+            "name": "time_stamp"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_offline_desktop b/ansible/roles/druid-ingestion/templates/raw_offline_desktop
new file mode 100644
index 0000000000..d31f22d320
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_offline_desktop
@@ -0,0 +1,1353 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "offline-desktop-events",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "root",
+              "name": "mid"
+            },
+            {
+              "type": "root",
+              "name": "syncts"
+            },
+            {
+              "type": "root",
+              "name": "@timestamp"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "actor_type",
+              "expr": "$.actor.type"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "context_sid",
+              "expr": "$.context.sid"
+            },
+            {
+              "type": "path",
+              "name": "context_did",
+              "expr": "$.context.did"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_type",
+              "expr": "$.context.cdata[*].type"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_id",
+              "expr": "$.context.cdata[*].id"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l1",
+              "expr": "$.context.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l2",
+              "expr": "$.context.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l3",
+              "expr": "$.context.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l4",
+              "expr": "$.context.rollup.l4"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l3",
+              "expr": "$.object.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l4",
+              "expr": "$.object.rollup.l4"
+            },
+            {
+              "type": "root",
+              "name": "tags"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_subtype",
+              "expr": "$.edata.subtype"
+            },
+            {
+              "type": "path",
+              "name": "edata_mode",
+              "expr": "$.edata.mode"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageid",
+              "expr": "$.edata.pageid"
+            },
+            {
+              "type": "path",
+              "name": "edata_uri",
+              "expr": "$.edata.uri"
+            },
+            {
+              "type": "path",
+              "name": "edata_id",
+              "expr": "$.edata.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "edata_index",
+              "expr": "$.edata.index"
+            },
+            {
+              "type": "path",
+              "name": "edata_pass",
+              "expr": "$.edata.pass"
+            },
+            {
+              "type": "path",
+              "name": "edata_score",
+              "expr": "$.edata.score"
+            },
+            {
+              "type": "path",
+              "name": "edata_resvalues",
+              "expr": "$.edata.resvalues[*]"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_id",
+              "expr": "$.edata.item.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_title",
+              "expr": "$.edata.item.title"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_maxscore",
+              "expr": "$.edata.item.maxscore"
+            },
+            {
+              "type": "path",
+              "name": "edata_target_id",
+              "expr": "$.edata.target.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_target_type",
+              "expr": "$.edata.target.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_rating",
+              "expr": "$.edata.rating"
+            },
+            {
+              "type": "path",
+              "name": "edata_comments",
+              "expr": "$.edata.comments"
+            },
+            {
+              "type": "path",
+              "name": "edata_dir",
+              "expr": "$.edata.dir"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_id",
+              "expr": "$.edata.items[*].id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_type",
+              "expr": "$.edata.items[*].type"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_origin_id",
+              "expr": "$.edata.items[*].origin.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_origin_type",
+              "expr": "$.edata.items[*].origin.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_to_id",
+              "expr": "$.edata.items[*].to.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_to_type",
+              "expr": "$.edata.items[*].to.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_id",
+              "expr": "$.edata.plugin.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_ver",
+              "expr": "$.edata.plugin.ver"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_category",
+              "expr": "$.edata.plugin.category"
+            },
+            {
+              "type": "path",
+              "name": "edata_props",
+              "expr": "$.edata.props[*]"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_prevstate",
+              "expr": "$.edata.prevstate"
+            },
+            {
+              "type": "path",
+              "name": "edata_size",
+              "expr": "$.edata.size"
+            },
+            {
+              "type": "path",
+              "name": "edata_filters_dialcodes",
+              "expr": "$.edata.filters.dialcodes"
+            },
+            {
+              "type": "path",
+              "name": "edata_topn_identifier",
+              "expr": "$.edata.topn[*].identifier"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objid",
+              "expr": "$.edata.visits[*].objid"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objtype",
+              "expr": "$.edata.visits[*].objtype"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objver",
+              "expr": "$.edata.visits[*].objver"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_index",
+              "expr": "$.edata.visits[*].index"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state",
+              "expr": "$.devicedata.state"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_code",
+              "expr": "$.devicedata.statecode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_city",
+              "expr": "$.devicedata.city"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country_code",
+              "expr": "$.devicedata.countrycode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country",
+              "expr": "$.devicedata.country"
+            },
+            {
+              "type": "path",
+              "name": "device_os",
+              "expr": "$.devicedata.devicespec.os"
+            },
+            {
+              "type": "path",
+              "name": "device_make",
+              "expr": "$.devicedata.devicespec.make"
+            },
+            {
+              "type": "path",
+              "name": "device_id",
+              "expr": "$.devicedata.devicespec.id"
+            },
+            {
+              "type": "path",
+              "name": "device_mem",
+              "expr": "$.devicedata.devicespec.mem"
+            },
+            {
+              "type": "path",
+              "name": "device_idisk",
+              "expr": "$.devicedata.devicespec.idisk"
+            },
+            {
+              "type": "path",
+              "name": "device_edisk",
+              "expr": "$.devicedata.devicespec.edisk"
+            },
+            {
+              "type": "path",
+              "name": "device_scrn",
+              "expr": "$.devicedata.devicespec.scrn"
+            },
+            {
+              "type": "path",
+              "name": "device_camera",
+              "expr": "$.devicedata.devicespec.camera"
+            },
+            {
+              "type": "path",
+              "name": "device_cpu",
+              "expr": "$.devicedata.devicespec.cpu"
+            },
+            {
+              "type": "path",
+              "name": "device_sims",
+              "expr": "$.devicedata.devicespec.sims"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_agent",
+              "expr": "$.devicedata.uaspec.agent"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_ver",
+              "expr": "$.devicedata.uaspec.ver"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_system",
+              "expr": "$.devicedata.uaspec.system"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_platform",
+              "expr": "$.devicedata.uaspec.platform"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_raw",
+              "expr": "$.devicedata.uaspec.raw"
+            },
+            {
+              "type": "path",
+              "name": "device_first_access",
+              "expr": "$.devicedata.firstaccess"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_code",
+              "expr": "$.devicedata.statecustomcode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_name",
+              "expr": "$.devicedata.statecustomname"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_district",
+              "expr": "$.devicedata.districtcustom"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_state",
+              "expr": "$.devicedata.userdeclared.state"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_district",
+              "expr": "$.devicedata.userdeclared.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_media_type",
+              "expr": "$.contentdata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_version",
+              "expr": "$.contentdata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "content_last_submitted_on",
+              "expr": "$.contentdata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_published_on",
+              "expr": "$.contentdata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_updated_on",
+              "expr": "$.contentdata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "content_created_by",
+              "expr": "$.contentdata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_object_type",
+              "expr": "$.collectiondata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_media_type",
+              "expr": "$.collectiondata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "collection_language",
+              "expr": "$.collectiondata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subjects",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_mimetype",
+              "expr": "$.collectiondata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "collection_framework",
+              "expr": "$.collectiondata.framework"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_status",
+              "expr": "$.collectiondata.status"
+            },
+            {
+              "type": "path",
+              "name": "collection_version",
+              "expr": "$.collectiondata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_submitted_on",
+              "expr": "$.collectiondata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_published_on",
+              "expr": "$.collectiondata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_updated_on",
+              "expr": "$.collectiondata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_by",
+              "expr": "$.collectiondata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_grade_list",
+              "expr": "$.userdata.gradelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_language_list",
+              "expr": "$.userdata.languagelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_subject_list",
+              "expr": "$.userdata.subjectlist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_roles",
+              "expr": "$.userdata.roles[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_state",
+              "expr": "$.userdata.state"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_district",
+              "expr": "$.userdata.district"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_batchcode",
+              "expr": "$.dialcodedata.batchcode"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_publisher",
+              "expr": "$.dialcodedata.publisher"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_generated_on",
+              "expr": "$.dialcodedata.generatedon"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_published_on",
+              "expr": "$.dialcodedata.publishedon"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_status",
+              "expr": "$.dialcodedata.status"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_object_type",
+              "expr": "$.dialcodedata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_iso_state_code",
+              "expr": "$.devicedata.iso3166statecode"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "mid"
+            },
+            {
+              "type": "long",
+              "name": "syncts"
+            },
+            {
+              "type": "string",
+              "name": "@timestamp"
+            },
+            {
+              "type": "string",
+              "name": "actor_id"
+            },
+            {
+              "type": "string",
+              "name": "actor_type"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "context_sid"
+            },
+            {
+              "type": "string",
+              "name": "context_did"
+            },
+            {
+              "name": "context_cdata_type"
+            },
+            {
+              "name": "context_cdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l4"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l4"
+            },
+            {
+              "name": "tags"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_subtype"
+            },
+            {
+              "type": "string",
+              "name": "edata_mode"
+            },
+            {
+              "type": "string",
+              "name": "edata_pageid"
+            },
+            {
+              "type": "string",
+              "name": "edata_uri"
+            },
+            {
+              "type": "string",
+              "name": "edata_id"
+            },
+            {
+              "type": "double",
+              "name": "edata_duration"
+            },
+            {
+              "type": "long",
+              "name": "edata_index"
+            },
+            {
+              "type": "string",
+              "name": "edata_pass"
+            },
+            {
+              "type": "double",
+              "name": "edata_score"
+            },
+            {
+              "name": "edata_resvalues"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_title"
+            },
+            {
+              "type": "double",
+              "name": "edata_item_maxscore"
+            },
+            {
+              "type": "string",
+              "name": "edata_target_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_target_type"
+            },
+            {
+              "type": "long",
+              "name": "edata_rating"
+            },
+            {
+              "type": "string",
+              "name": "edata_comments"
+            },
+            {
+              "type": "string",
+              "name": "edata_dir"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_origin_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_origin_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_to_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_to_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_ver"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_category"
+            },
+            {
+              "name": "edata_props"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "string",
+              "name": "edata_prevstate"
+            },
+            {
+              "type": "long",
+              "name": "edata_size"
+            },
+            {
+              "name": "edata_filters_dialcodes"
+            },
+            {
+              "type": "string",
+              "name": "edata_topn_identifier"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objid"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objtype"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objver"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_index"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_city"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country"
+            },
+            {
+              "type": "string",
+              "name": "device_os"
+            },
+            {
+              "type": "string",
+              "name": "device_make"
+            },
+            {
+              "type": "string",
+              "name": "device_id"
+            },
+            {
+              "type": "long",
+              "name": "device_mem"
+            },
+            {
+              "type": "string",
+              "name": "device_idisk"
+            },
+            {
+              "type": "string",
+              "name": "device_edisk"
+            },
+            {
+              "type": "string",
+              "name": "device_scrn"
+            },
+            {
+              "type": "string",
+              "name": "device_camera"
+            },
+            {
+              "type": "string",
+              "name": "device_cpu"
+            },
+            {
+              "type": "long",
+              "name": "device_sims"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_agent"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_ver"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_system"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_platform"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_raw"
+            },
+            {
+              "type": "long",
+              "name": "device_first_access"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_name"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_state"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "type": "string",
+              "name": "content_media_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "double",
+              "name": "content_version"
+            },
+            {
+              "type": "long",
+              "name": "content_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "content_created_by"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_object_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_media_type"
+            },
+            {
+              "name": "collection_language"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subjects"
+            },
+            {
+              "type": "string",
+              "name": "collection_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "collection_framework"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_status"
+            },
+            {
+              "type": "double",
+              "name": "collection_version"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_by"
+            },
+            {
+              "name": "collection_created_for"
+            },
+            {
+              "name": "user_grade_list"
+            },
+            {
+              "name": "user_language_list"
+            },
+            {
+              "name": "user_subject_list"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "name": "user_roles"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_batchcode"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_publisher"
+            },
+            {
+              "type": "long",
+              "name": "dialcode_generated_on"
+            },
+            {
+              "type": "long",
+              "name": "dialcode_published_on"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_status"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_object_type"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_iso_state_code"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "ets",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "day",
+      "rollup": false
+    },
+    "transformSpec": {
+      "filter": {
+        "type": "selector",
+        "dimension": "context_pdata_id",
+        "value": "{{desktop_pdata_id}}"
+      }
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{raw_offline_desktop_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": false,
+    "completionTimeout" : "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 2000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation b/ansible/roles/druid-ingestion/templates/raw_sl_observation
new file mode 100644
index 0000000000..f2a0a7da72
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation
@@ -0,0 +1,507 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "sl-observation",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "completedDate",
+          "format": "auto"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "completedDate"
+            },
+            {
+              "type": "string",
+              "name": "createdAt"
+            },
+            {
+              "type": "string",
+              "name": "createdBy"
+            },
+            {
+              "type": "string",
+              "name": "criteriaExternalId"
+            },
+            {
+              "type": "string",
+              "name": "criteriaId"
+            },
+            {
+              "type": "string",
+              "name": "criteriaName"
+            },
+            {
+              "type": "string",
+              "name": "entityType"
+            },
+            {
+              "type": "string",
+              "name": "entityTypeId"
+            },
+            {
+              "type": "string",
+              "name": "observationId"
+            },
+            {
+              "type": "string",
+              "name": "observationName"
+            },
+            {
+              "type": "string",
+              "name": "observationSubmissionId"
+            },
+            {
+              "type": "string",
+              "name": "questionAnswer"
+            },
+            {
+              "type": "string",
+              "name": "questionECM"
+            },
+            {
+              "type": "string",
+              "name": "questionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "questionId"
+            },
+            {
+              "type": "string",
+              "name": "questionName"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseLabel"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseType"
+            },
+            {
+              "type": "string",
+              "name": "solutionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "solutionId"
+            },
+            {
+              "type": "string",
+              "name": "solutionName"
+            },
+            {
+              "type": "string",
+              "name": "updatedAt"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentId"
+            },
+            {
+              "type": "string",
+              "name": "instanceId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentResponsetype"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentQuestion"
+            },
+            {
+              "type": "string",
+              "name": "questionSequenceByEcm"
+            },
+            {
+              "type": "string",
+              "name": "maxScore"
+            },
+            {
+              "type": "string",
+              "name": "minScore"
+            },
+            {
+              "type": "string",
+              "name": "percentageScore"
+            },
+            {
+              "type": "string",
+              "name": "pointsBasedScoreInParent"
+            },
+            {
+              "type": "string",
+              "name": "totalScore"
+            },
+            {
+              "type": "string",
+              "name": "scoreAchieved"
+            },
+            {
+              "type": "string",
+              "name": "totalpercentage"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentExternalId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentEcmSequence"
+            },
+            {
+              "type": "string",
+              "name": "remarks"
+            },
+            {
+              "type": "string",
+              "name": "total_evidences"
+            },
+            {
+              "type": "string",
+              "name": "evidence_count"
+            },
+            {
+              "type": "string",
+              "name": "school"
+            },
+            {
+              "type": "string",
+              "name": "block"
+            },
+            {
+              "type": "string",
+              "name": "district"
+            },
+            {
+              "type": "string",
+              "name": "cluster"
+            },
+            {
+              "type": "string",
+              "name": "state"
+            },
+            {
+              "type": "string",
+              "name": "schoolName"
+            },
+            {
+              "type": "string",
+              "name": "blockName"
+            },
+            {
+              "type": "string",
+              "name": "districtName"
+            },
+            {
+              "type": "string",
+              "name": "clusterName"
+            },
+            {
+              "type": "string",
+              "name": "stateName"
+            },
+            {
+              "type": "string",
+              "name": "schoolExternalId"
+            },
+            {
+              "type": "string",
+              "name": "blockExternalId"
+            },
+            {
+              "type": "string",
+              "name": "districtExternalId"
+            },
+            {
+              "type": "string",
+              "name": "clusterExternalId"
+            },
+            {
+              "type": "string",
+              "name": "stateExternalId"
+            },
+            {
+              "type": "string",
+              "name": "schoolTypes"
+            },
+            {
+              "type": "string",
+              "name": "administrationTypes"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaExternalId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaName"
+            },
+            {
+              "type": "string",
+              "name": "role_title"
+            },
+            {
+              "type": "string",
+              "name": "entity"
+            },
+            {
+              "type": "string",
+              "name": "entityExternalId"
+            },
+            {
+              "type": "string",
+              "name": "entityName"
+            },
+            {
+              "type": "string",
+              "name": "isAPrivateProgram"
+            },
+            {
+              "type": "string",
+              "name": "programId"
+            },
+            {
+              "type": "string",
+              "name": "programName"
+            },
+            {
+              "type": "string",
+              "name": "programExternalId"
+            },
+            {
+              "name": "questionResponseLabel_number",
+              "type": "float"
+            },
+            {
+              "type": "string",
+              "name": "criteriaLevel"
+            },
+            {
+              "type": "string",
+              "name": "criteriaScore"
+            },
+            {
+              "type": "string",
+              "name": "submissionNumber"
+            },
+            {
+              "type": "string",
+              "name": "submissionTitle"
+            },
+            {
+              "type": "string",
+              "name": "channel"
+            },
+            {
+              "type": "string",
+              "name": "parent_channel"
+            },
+            {
+              "type": "string",
+              "name": "user_districtName"
+            },
+            {
+              "type": "string",
+              "name": "user_blockName"
+            },
+            {
+              "type": "string",
+              "name": "user_clusterName"
+            },
+            {
+              "type": "string",
+              "name": "appName"
+            },
+            {
+              "type": "string",
+              "name": "evidences"
+            },
+            {
+              "type": "string",
+              "name": "user_stateName"
+            },
+            {
+              "type": "string",
+              "name": "domainName"
+            },
+            {
+              "type": "string",
+              "name": "domainExternalId"
+            },
+            {
+              "type": "string",
+              "name": "childName"
+            },
+            {
+              "type": "string",
+              "name": "childType"
+            },
+            {
+              "type": "string",
+              "name": "childExternalid"
+            },
+            {
+              "type": "string",
+              "name": "level"
+            },
+            {
+              "type": "string",
+              "name": "criteriaDescription"
+            },
+            {
+              "type": "string",
+              "name": "programDescription"
+            },
+            {
+              "type": "string",
+              "name": "solutionDescription"
+            },
+            {
+              "type": "string",
+              "name": "label"
+            },
+            {
+              "type": "string",
+              "name": "imp_project_id"
+            },
+            {
+              "type": "string",
+              "name": "imp_project_title"
+            },
+            {
+              "type": "string",
+              "name": "imp_project_goal"
+            },
+            {
+              "type": "string",
+              "name": "imp_project_externalId"
+            },
+            {
+              "type": "string",
+              "name": "ancestorName"
+            },
+            {
+              "type": "string",
+              "name": "scoringSystem"
+            },
+            {
+              "type": "string",
+              "name": "domainLevel"
+            },
+            {
+              "type": "string",
+              "name": "domainScore"
+            },
+            {
+              "name": "criteriaLevelReport",
+              "type": "boolean"
+            },
+            {
+              "type": "string",
+              "name": "user_schoolName"
+            },
+            {
+              "type": "string",
+              "name": "school_externalId"
+            },
+            {
+              "type": "string",
+              "name": "school_code"
+            },
+            {
+              "type": "string",
+              "name": "solution_type"
+            },
+            {
+              "type": "string",
+              "name": "organisation_name"
+            },
+            {
+              "type": "string",
+              "name": "user_boardName"
+            },
+            {
+              "type": "string",
+              "name": "district_externalId"
+            },
+            {
+              "type": "string",
+              "name": "state_externalId"
+            },
+            {
+              "type": "string",
+              "name": "block_externalId"
+            },
+            {
+              "type": "string",
+              "name": "cluster_externalId"
+            },
+            {
+              "type": "string",
+              "name": "organisation_id"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "state_code"
+            },
+            {
+              "type": "string",
+              "name": "block_code"
+            },
+            {
+              "type": "string",
+              "name": "district_code"
+            },
+            {
+              "type": "string",
+              "name": "cluster_code"
+            }
+          ]
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "floatSum",
+        "name": "question_response_number",
+        "fieldName": "questionResponseLabel_number"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": "none",
+      "rollup": false
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "{{env}}.ml.observation.druid",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": "{{ml_observation_task_count}}",
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence b/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence
new file mode 100644
index 0000000000..9e2122ebc3
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_evidence
@@ -0,0 +1,125 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "sl-observation-evidence",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "completedDate",
+          "format": "auto"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "total_evidences"
+            },
+            {
+              "type": "string",
+              "name": "observationSubmissionId"
+            },
+            {
+              "type": "string",
+              "name": "entity"
+            },
+            {
+              "type": "string",
+              "name": "entityExternalId"
+            },
+            {
+              "type": "string",
+              "name": "entityName"
+            },
+            {
+              "type": "string",
+              "name": "entityTypeId"
+            },
+            {
+              "type": "string",
+              "name": "entityType"
+            },
+            {
+              "type": "string",
+              "name": "createdBy"
+            },
+            {
+              "type": "string",
+              "name": "solutionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "solutionId"
+            },
+            {
+              "type": "string",
+              "name": "observationId"
+            },
+            {
+              "type": "string",
+              "name": "remarks"
+            },
+            {
+              "type": "string",
+              "name": "questionId"
+            },
+            {
+              "type": "string",
+              "name": "questionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseType"
+            },
+            {
+              "type": "string",
+              "name": "questionName"
+            },
+            {
+              "type": "string",
+              "name": "evidence_count"
+            },
+            {
+              "type": "string",
+              "name": "fileName"
+            },
+            {
+              "type": "string",
+              "name": "fileSourcePath"
+            },
+            {
+              "type": "string",
+              "name": "completedDate"
+            },
+            {
+              "type": "string",
+              "name": "appName"
+            }
+          ]
+        }
+      }
+    },
+    "metricsSpec": [],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": "none",
+      "rollup": false
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "{{env}}.ml.observation.evidence.druid",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": "{{ml_observation_evidence_task_count}}",
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_observation_status b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status
new file mode 100644
index 0000000000..2d49030ca7
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_observation_status
@@ -0,0 +1,209 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/observation/status/sl_observation_status.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "sl-observation-status",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "updatedAt",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "status"
+          },
+          {
+            "type": "string",
+            "name": "entity_externalId"
+          },
+          {
+            "type": "string",
+            "name": "entity_id"
+          },
+          {
+            "type": "string",
+            "name": "entity_type"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "solution_externalId"
+          },
+          {
+            "type": "string",
+            "name": "submission_id"
+          },
+          {
+            "type": "string",
+            "name": "entity_name"
+          },
+          {
+            "type": "string",
+            "name": "solution_name"
+          },
+          {
+            "type": "string",
+            "name": "role_title"
+          },
+          {
+            "type": "string",
+            "name": "school_name"
+          },
+          {
+            "type": "string",
+            "name": "school_code"
+          },
+          {
+            "type": "string",
+            "name": "school_externalId"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_code"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_code"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "block_name"
+          },
+          {
+            "type": "string",
+            "name": "block_code"
+          },
+          {
+            "type": "string",
+            "name": "block_externalId"
+          },
+          {
+            "type": "string",
+            "name": "cluster_name"
+          },
+          {
+            "type": "string",
+            "name": "cluster_code"
+          },
+          {
+            "type": "string",
+            "name": "cluster_externalId"
+          },
+          {
+            "type": "string",
+            "name": "completedDate"
+          },
+          {
+            "type": "string",
+            "name": "channel"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "program_externalId"
+          },
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "app_name"
+          },
+          {
+            "type": "string",
+            "name": "user_id"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "solution_type"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "ecm_marked_na"
+          },
+          {
+            "type": "string",
+            "name": "board_name"
+          },
+          {
+            "type": "string",
+            "name": "updatedAt"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "user_type"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_project_index b/ansible/roles/druid-ingestion/templates/raw_sl_project_index
new file mode 100644
index 0000000000..1e6881d72d
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_project_index
@@ -0,0 +1,297 @@
+{
+  "type": "index",
+  "spec": {
+    "ioConfig": {
+      "type": "index",
+      "firehose": {
+        "type": "static-azure-blobstore",
+        "blobs": [
+          {
+            "container": "{{container_name}}",
+            "path": "/projects/sl_projects.json"
+          }
+        ],
+        "fetchTimeout": 300000
+      },
+      "inputFormat": {
+        "type": "json"
+      }
+    },
+    "tuningConfig": {
+      "type": "index",
+      "partitionsSpec": {
+        "type": "dynamic"
+      }
+    },
+    "dataSchema": {
+      "dataSource": "sl-project",
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "none",
+        "rollup": false,
+        "segmentGranularity": "DAY"
+      },
+      "timestampSpec": {
+        "column": "project_last_sync",
+        "format": "auto"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          {
+            "type": "string",
+            "name": "school_name"
+          },
+          {
+            "type": "string",
+            "name": "school_code"
+          },
+          {
+            "type": "string",
+            "name": "school_externalId"
+          },
+          {
+            "type": "string",
+            "name": "block_name"
+          },
+          {
+            "type": "string",
+            "name": "block_code"
+          },
+          {
+            "type": "string",
+            "name": "block_externalId"
+          },
+          {
+            "type": "string",
+            "name": "district_name"
+          },
+          {
+            "type": "string",
+            "name": "district_code"
+          },
+          {
+            "type": "string",
+            "name": "district_externalId"
+          },
+          {
+            "type": "string",
+            "name": "cluster_name"
+          },
+          {
+            "type": "string",
+            "name": "cluster_code"
+          },
+          {
+            "type": "string",
+            "name": "cluster_externalId"
+          },
+          {
+            "type": "string",
+            "name": "state_name"
+          },
+          {
+            "type": "string",
+            "name": "state_code"
+          },
+          {
+            "type": "string",
+            "name": "state_externalId"
+          },
+          {
+            "type": "string",
+            "name": "project_title"
+          },
+          {
+            "type": "string",
+            "name": "project_goal"
+          },
+          {
+            "type": "string",
+            "name": "project_last_sync"
+          },
+          {
+            "type": "string",
+            "name": "area_of_improvement"
+          },
+          {
+            "type": "string",
+            "name": "status_of_project"
+          },
+          {
+            "type": "string",
+            "name": "tasks"
+          },
+          {
+            "type": "string",
+            "name": "tasks_date"
+          },
+          {
+            "type": "string",
+            "name": "tasks_status"
+          },
+          {
+            "type": "string",
+            "name": "sub_task"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_status"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_date"
+          },
+          {
+            "type": "string",
+            "name": "task_start_date"
+          },
+          {
+            "type": "string",
+            "name": "task_end_date"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_start_date"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_end_date"
+          },
+          {
+            "type": "string",
+            "name": "designation"
+          },
+          {
+            "type": "string",
+            "name": "task_evidence"
+          },
+          {
+            "type": "string",
+            "name": "task_evidence_status"
+          },
+          {
+            "type": "string",
+            "name": "project_id"
+          },
+          {
+            "type": "string",
+            "name": "task_id"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_id"
+          },
+          {
+            "type": "string",
+            "name": "project_created_type"
+          },
+          {
+            "type": "string",
+            "name": "task_assigned_to"
+          },
+          {
+            "type": "string",
+            "name": "channel"
+          },
+          {
+            "type": "string",
+            "name": "parent_channel"
+          },
+          {
+            "type": "string",
+            "name": "program_id"
+          },
+          {
+            "type": "string",
+            "name": "program_name"
+          },
+          {
+            "type": "string",
+            "name": "project_created_date"
+          },
+          {
+            "type": "string",
+            "name": "project_deleted_flag"
+          },
+          {
+            "type": "string",
+            "name": "project_updated_date"
+          },
+          {
+            "type": "string",
+            "name": "createdBy"
+          },
+          {
+            "type": "string",
+            "name": "project_title_editable"
+          },
+          {
+            "type": "string",
+            "name": "project_duration"
+          },
+          {
+            "type": "string",
+            "name": "program_externalId"
+          },
+          {
+            "type": "string",
+            "name": "private_program"
+          },
+          {
+            "type": "string",
+            "name": "task_deleted_flag"
+          },
+          {
+            "type": "string",
+            "name": "sub_task_deleted_flag"
+          },
+          {
+            "type": "string",
+            "name": "project_terms_and_condition"
+          },
+          {
+            "type": "string",
+            "name": "task_remarks"
+          },
+          {
+            "type": "string",
+            "name": "organisation_name"
+          },
+          {
+            "type": "string",
+            "name": "project_description"
+          },
+          {
+            "type": "string",
+            "name": "project_completed_date"
+          },
+          {
+            "type": "string",
+            "name": "solution_id"
+          },
+          {
+            "type": "string",
+            "name": "project_remarks"
+          },
+          {
+            "type": "string",
+            "name": "project_evidence"
+          },
+          {
+            "type": "string",
+            "name": "board_name"
+          },
+          {
+            "type": "string",
+            "name": "organisation_id"
+          },
+          {
+            "type": "string",
+            "name": "user_type"
+          }
+        ]
+      },
+      "metricsSpec": []
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey b/ansible/roles/druid-ingestion/templates/raw_sl_survey
new file mode 100644
index 0000000000..18719f9a7f
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey
@@ -0,0 +1,319 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "sl-survey",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "completedDate",
+          "format": "auto"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "completedDate"
+            },
+            {
+              "type": "string",
+              "name": "createdAt"
+            },
+            {
+              "type": "string",
+              "name": "createdBy"
+            },
+            {
+              "type": "string",
+              "name": "criteriaExternalId"
+            },
+            {
+              "type": "string",
+              "name": "criteriaId"
+            },
+            {
+              "type": "string",
+              "name": "criteriaName"
+            },
+            {
+              "type": "string",
+              "name": "surveyId"
+            },
+            {
+              "type": "string",
+              "name": "surveyName"
+            },
+            {
+              "type": "string",
+              "name": "surveySubmissionId"
+            },
+            {
+              "type": "string",
+              "name": "questionAnswer"
+            },
+            {
+              "type": "string",
+              "name": "questionECM"
+            },
+            {
+              "type": "string",
+              "name": "questionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "questionId"
+            },
+            {
+              "type": "string",
+              "name": "questionName"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseLabel"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseType"
+            },
+            {
+              "type": "string",
+              "name": "solutionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "solutionId"
+            },
+            {
+              "type": "string",
+              "name": "solutionName"
+            },
+            {
+              "type": "string",
+              "name": "updatedAt"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentId"
+            },
+            {
+              "type": "string",
+              "name": "instanceId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentResponsetype"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentQuestion"
+            },
+            {
+              "type": "string",
+              "name": "questionSequenceByEcm"
+            },
+            {
+              "type": "string",
+              "name": "maxScore"
+            },
+            {
+              "type": "string",
+              "name": "minScore"
+            },
+            {
+              "type": "string",
+              "name": "percentageScore"
+            },
+            {
+              "type": "string",
+              "name": "pointsBasedScoreInParent"
+            },
+            {
+              "type": "string",
+              "name": "totalScore"
+            },
+            {
+              "type": "string",
+              "name": "scoreAchieved"
+            },
+            {
+              "type": "string",
+              "name": "totalpercentage"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentExternalId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentEcmSequence"
+            },
+            {
+              "type": "string",
+              "name": "remarks"
+            },
+            {
+              "type": "string",
+              "name": "total_evidences"
+            },
+            {
+              "type": "string",
+              "name": "evidence_count"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaExternalId"
+            },
+            {
+              "type": "string",
+              "name": "instanceParentCriteriaName"
+            },
+            {
+              "type": "string",
+              "name": "isAPrivateProgram"
+            },
+            {
+              "type": "string",
+              "name": "programId"
+            },
+            {
+              "type": "string",
+              "name": "programName"
+            },
+            {
+              "type": "string",
+              "name": "programExternalId"
+            },
+            {
+              "name": "questionResponseLabel_number",
+              "type": "float"
+            },
+            {
+              "type": "string",
+              "name": "channel"
+            },
+            {
+              "type": "string",
+              "name": "parent_channel"
+            },
+            {
+              "type": "string",
+              "name": "appName"
+            },
+            {
+              "type": "string",
+              "name": "organisation_name"
+            },
+              {
+                "type": "string",
+                "name": "user_subtype"
+              },
+              {
+                "type": "string",
+                "name": "user_type"
+              },
+              {
+                "type": "string",
+                "name": "board_name"
+              },
+              {
+                "type": "string",
+                "name": "district_code"
+              },
+              {
+                "type": "string",
+                "name": "district_name"
+              },
+              {
+                "type": "string",
+                "name": "district_externalId"
+              },
+              {
+                "type": "string",
+                "name": "block_code"
+              },
+              {
+                "type": "string",
+                "name": "block_name"
+              },
+              {
+                "type": "string",
+                "name": "block_externalId"
+              },
+              {
+                "type": "string",
+                "name": "school_code"
+              },
+              {
+                "type": "string",
+                "name": "school_name"
+              },
+              {
+                "type": "string",
+                "name": "school_externalId"
+              },
+              {
+                "type": "string",
+                "name": "cluster_code"
+              },
+              {
+                "type": "string",
+                "name": "cluster_name"
+              },
+              {
+                "type": "string",
+                "name": "cluster_externalId"
+              },
+              {
+                "type": "string",
+                "name": "state_code"
+              },
+              {
+                "type": "string",
+                "name": "state_name"
+              },
+              {
+                "type": "string",
+                "name": "state_externalId"
+              },
+              {
+                "type": "string",
+                "name": "organisation_id"
+              }
+          ]
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "floatSum",
+        "name": "question_response_number",
+        "fieldName": "questionResponseLabel_number"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": "none",
+      "rollup": false
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "{{env}}.ml.survey.druid",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": "{{ml_survey_task_count}}",
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_sl_survey_evidence b/ansible/roles/druid-ingestion/templates/raw_sl_survey_evidence
new file mode 100644
index 0000000000..c2adcc6332
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_sl_survey_evidence
@@ -0,0 +1,105 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "sl-survey-evidence",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "timestampSpec": {
+          "column": "completedDate",
+          "format": "auto"
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "total_evidences"
+            },
+            {
+              "type": "string",
+              "name": "surveySubmissionId"
+            },
+            {
+              "type": "string",
+              "name": "createdBy"
+            },
+            {
+              "type": "string",
+              "name": "solutionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "solutionId"
+            },
+            {
+              "type": "string",
+              "name": "surveyId"
+            },
+            {
+              "type": "string",
+              "name": "remarks"
+            },
+            {
+              "type": "string",
+              "name": "questionId"
+            },
+            {
+              "type": "string",
+              "name": "questionExternalId"
+            },
+            {
+              "type": "string",
+              "name": "questionResponseType"
+            },
+            {
+              "type": "string",
+              "name": "questionName"
+            },
+            {
+              "type": "string",
+              "name": "evidence_count"
+            },
+            {
+              "type": "string",
+              "name": "fileName"
+            },
+            {
+              "type": "string",
+              "name": "fileSourcePath"
+            },
+            {
+              "type": "string",
+              "name": "completedDate"
+            },
+            {
+              "type": "string",
+              "name": "appName"
+            }
+          ]
+        }
+      }
+    },
+    "metricsSpec": [],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "queryGranularity": "none",
+      "rollup": false
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false
+  },
+  "ioConfig": {
+    "topic": "{{env}}.ml.survey.evidence.druid",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": "{{ml_survey_evidence_task_count}}",
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_summary_events b/ansible/roles/druid-ingestion/templates/raw_summary_events
new file mode 100644
index 0000000000..1251b82920
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_summary_events
@@ -0,0 +1,1211 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "summary-events",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "root",
+              "name": "mid"
+            },
+            {
+              "type": "root",
+              "name": "ver"
+            },
+            {
+              "type": "root",
+              "name": "ets"
+            },
+            {
+              "type": "root",
+              "name": "uid"
+            },
+            {
+              "type": "root",
+              "name": "syncts"
+            },
+            {
+              "type": "path",
+              "name": "context_date_range_from",
+              "expr": "$.context.date_range.from"
+            },
+            {
+              "type": "path",
+              "name": "context_date_range_to",
+              "expr": "$.context.date_range.to"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_type",
+              "expr": "$.context.cdata[*].type"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_id",
+              "expr": "$.context.cdata[*].id"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l1",
+              "expr": "$.context.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l2",
+              "expr": "$.context.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l3",
+              "expr": "$.context.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l4",
+              "expr": "$.context.rollup.l4"
+            },
+            {
+              "type": "path",
+              "name": "dimension_channel",
+              "expr": "$.dimensions.channel"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_did",
+              "expr": "$.dimensions.did"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_id",
+              "expr": "$.dimensions.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_pid",
+              "expr": "$.dimensions.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_ver",
+              "expr": "$.dimensions.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_sid",
+              "expr": "$.dimensions.sid"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_type",
+              "expr": "$.dimensions.type"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_mode",
+              "expr": "$.dimensions.mode"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l3",
+              "expr": "$.object.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l4",
+              "expr": "$.object.rollup.l4"
+            },
+            {
+              "type": "root",
+              "name": "tags"
+            },
+            {
+              "type": "path",
+              "name": "edata_time_spent",
+              "expr": "$.edata.eks.time_spent"
+            },
+            {
+              "type": "path",
+              "name": "edata_time_difference",
+              "expr": "$.edata.eks.time_diff"
+            },
+            {
+              "type": "path",
+              "name": "edata_interaction_count",
+              "expr": "$.edata.eks.interact_events_count"
+            },
+            {
+              "type": "path",
+              "name": "edata_env_summary_env",
+              "expr": "$.edata.eks.env_summary[*].env"
+            },
+            {
+              "type": "path",
+              "name": "edata_env_summary_count",
+              "expr": "$.edata.eks.env_summary[*].count"
+            },
+            {
+              "type": "path",
+              "name": "edata_env_summary_time_spent",
+              "expr": "$.edata.eks.env_summary[*].time_spent"
+            },
+            {
+              "type": "path",
+              "name": "edata_page_summary_id",
+              "expr": "$.edata.eks.page_summary[*].id"
+            },
+            {
+              "type": "path",
+              "name": "edata_page_summary_type",
+              "expr": "$.edata.eks.page_summary[*].type"
+            },
+            {
+              "type": "path",
+              "name": "edata_page_summary_env",
+              "expr": "$.edata.eks.page_summary[*].env"
+            },
+            {
+              "type": "path",
+              "name": "edata_page_summary_visit_count",
+              "expr": "$.edata.eks.page_summary[*].visit_count"
+            },
+            {
+              "type": "path",
+              "name": "edata_page_summary_time_spent",
+              "expr": "$.edata.eks.page_summary[*].time_spent"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_item_id",
+              "expr": "$.edata.eks.item_responses[*].itemId"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_time_spent",
+              "expr": "$.edata.eks.item_responses[*].timeSpent"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_pass",
+              "expr": "$.edata.eks.item_responses[*].pass"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_score",
+              "expr": "$.edata.eks.item_responses[*].score"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_max_score",
+              "expr": "$.edata.eks.item_responses[*].maxScore"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_responses_timestamp",
+              "expr": "$.edata.eks.item_responses[*].time_stamp"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state",
+              "expr": "$.devicedata.state"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_code",
+              "expr": "$.devicedata.statecode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_city",
+              "expr": "$.devicedata.city"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country_code",
+              "expr": "$.devicedata.countrycode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country",
+              "expr": "$.devicedata.country"
+            },
+            {
+              "type": "path",
+              "name": "device_os",
+              "expr": "$.devicedata.devicespec.os"
+            },
+            {
+              "type": "path",
+              "name": "device_make",
+              "expr": "$.devicedata.devicespec.make"
+            },
+            {
+              "type": "path",
+              "name": "device_id",
+              "expr": "$.devicedata.devicespec.id"
+            },
+            {
+              "type": "path",
+              "name": "device_mem",
+              "expr": "$.devicedata.devicespec.mem"
+            },
+            {
+              "type": "path",
+              "name": "device_idisk",
+              "expr": "$.devicedata.devicespec.idisk"
+            },
+            {
+              "type": "path",
+              "name": "device_edisk",
+              "expr": "$.devicedata.devicespec.edisk"
+            },
+            {
+              "type": "path",
+              "name": "device_scrn",
+              "expr": "$.devicedata.devicespec.scrn"
+            },
+            {
+              "type": "path",
+              "name": "device_camera",
+              "expr": "$.devicedata.devicespec.camera"
+            },
+            {
+              "type": "path",
+              "name": "device_cpu",
+              "expr": "$.devicedata.devicespec.cpu"
+            },
+            {
+              "type": "path",
+              "name": "device_sims",
+              "expr": "$.devicedata.devicespec.sims"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_agent",
+              "expr": "$.devicedata.uaspec.agent"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_ver",
+              "expr": "$.devicedata.uaspec.ver"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_system",
+              "expr": "$.devicedata.uaspec.system"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_platform",
+              "expr": "$.devicedata.uaspec.platform"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_raw",
+              "expr": "$.devicedata.uaspec.raw"
+            },
+            {
+              "type": "path",
+              "name": "device_first_access",
+              "expr": "$.devicedata.firstaccess"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_code",
+              "expr": "$.devicedata.statecustomcode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_name",
+              "expr": "$.devicedata.statecustomname"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_district",
+              "expr": "$.devicedata.districtcustom"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_state",
+              "expr": "$.devicedata.userdeclared.state"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_district",
+              "expr": "$.devicedata.userdeclared.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_media_type",
+              "expr": "$.contentdata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_created_by",
+              "expr": "$.contentdata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "content_version",
+              "expr": "$.contentdata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "content_last_submitted_on",
+              "expr": "$.contentdata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_published_on",
+              "expr": "$.contentdata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_updated_on",
+              "expr": "$.contentdata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_object_type",
+              "expr": "$.collectiondata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_media_type",
+              "expr": "$.collectiondata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "collection_language",
+              "expr": "$.collectiondata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subjects",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_mimetype",
+              "expr": "$.collectiondata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "collection_framework",
+              "expr": "$.collectiondata.framework"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_status",
+              "expr": "$.collectiondata.status"
+            },
+            {
+              "type": "path",
+              "name": "collection_version",
+              "expr": "$.collectiondata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_submitted_on",
+              "expr": "$.collectiondata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_published_on",
+              "expr": "$.collectiondata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_updated_on",
+              "expr": "$.collectiondata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_by",
+              "expr": "$.collectiondata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_grade_list",
+              "expr": "$.userdata.gradelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_language_list",
+              "expr": "$.userdata.languagelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_subject_list",
+              "expr": "$.userdata.subjectlist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_roles",
+              "expr": "$.userdata.roles[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_state",
+              "expr": "$.userdata.state"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_district",
+              "expr": "$.userdata.district"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_iso_state_code",
+              "expr": "$.devicedata.iso3166statecode"
+            },
+{
+              "type": "path",
+              "name": "object_l2_created_for",
+              "expr": "$.l2data.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_framework",
+              "expr": "$.l2data.framework"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_name",
+              "expr": "$.l2data.name"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_channel",
+              "expr": "$.l2data.channel"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_mimetype",
+              "expr": "$.l2data.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_medium",
+              "expr": "$.l2data.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_board",
+              "expr": "$.l2data.board"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_contenttype",
+              "expr": "$.l2data.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_gradelevel",
+              "expr": "$.l2data.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_subjects",
+              "expr": "$.l2data.subject[*]"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "mid"
+            },
+            {
+              "type": "string",
+              "name": "ver"
+            },
+            {
+              "type": "long",
+              "name": "syncts"
+            },
+            {
+              "type": "string",
+              "name": "uid"
+            },
+            {
+              "type": "long",
+              "name": "context_date_range_from"
+            },
+            {
+              "name": "context_cdata_type"
+            },
+            {
+              "name": "context_cdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l4"
+            },
+            {
+              "type": "string",
+              "name": "dimension_channel"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_did"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_sid"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_type"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_mode"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l4"
+            },
+            {
+              "name": "tags"
+            },
+            {
+              "type": "double",
+              "name": "edata_time_spent"
+            },
+            {
+              "type": "double",
+              "name": "edata_time_difference"
+            },
+            {
+              "type": "long",
+              "name": "edata_interaction_count"
+            },
+            {
+              "type": "string",
+              "name": "edata_env_summary_env"
+            },
+            {
+              "type": "string",
+              "name": "edata_env_summary_count"
+            },
+            {
+              "type": "string",
+              "name": "edata_env_summary_time_spent"
+            },
+            {
+              "type": "string",
+              "name": "edata_page_summary_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_page_summary_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_page_summary_env"
+            },
+            {
+              "type": "string",
+              "name": "edata_page_summary_visit_count"
+            },
+            {
+              "type": "string",
+              "name": "edata_page_summary_time_spent"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_item_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_time_spent"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_pass"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_score"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_max_score"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_responses_timestamp"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_city"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country"
+            },
+            {
+              "type": "string",
+              "name": "device_os"
+            },
+            {
+              "type": "string",
+              "name": "device_make"
+            },
+            {
+              "type": "string",
+              "name": "device_id"
+            },
+            {
+              "type": "long",
+              "name": "device_mem"
+            },
+            {
+              "type": "string",
+              "name": "device_idisk"
+            },
+            {
+              "type": "string",
+              "name": "device_edisk"
+            },
+            {
+              "type": "string",
+              "name": "device_scrn"
+            },
+            {
+              "type": "string",
+              "name": "device_camera"
+            },
+            {
+              "type": "string",
+              "name": "device_cpu"
+            },
+            {
+              "type": "long",
+              "name": "device_sims"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_agent"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_ver"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_system"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_platform"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_raw"
+            },
+            {
+              "type": "long",
+              "name": "device_first_access"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_name"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_state"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "type": "string",
+              "name": "content_media_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "double",
+              "name": "content_version"
+            },
+            {
+              "type": "long",
+              "name": "content_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "content_created_by"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_object_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_media_type"
+            },
+            {
+              "name": "collection_language"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subjects"
+            },
+            {
+              "type": "string",
+              "name": "collection_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "collection_framework"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_status"
+            },
+            {
+              "type": "double",
+              "name": "collection_version"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_by"
+            },
+            {
+              "name": "collection_created_for"
+            },
+            {
+              "name": "user_grade_list"
+            },
+            {
+              "name": "user_language_list"
+            },
+            {
+              "name": "user_subject_list"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "name": "user_roles"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_iso_state_code"
+            },
+            {
+              "name": "object_l2_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_framework"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_name"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_channel"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_mimetype"
+            },
+            {
+              "name": "object_l2_medium"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_board"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_contenttype"
+            },
+            {
+              "name": "object_l2_gradelevel"
+            },
+            {
+              "name": "object_l2_subjects"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "longSum",
+        "name": "total_interactions",
+        "fieldName": "edata_interaction_count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_time_spent",
+        "fieldName": "edata_time_spent"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "day",
+      "rollup": true
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.summary",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{raw_summary_events_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true,
+    "completionTimeout": "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_telemetry_events b/ansible/roles/druid-ingestion/templates/raw_telemetry_events
new file mode 100644
index 0000000000..384a9df5fc
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_telemetry_events
@@ -0,0 +1,1568 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "telemetry-events-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "root",
+              "name": "mid"
+            },
+            {
+              "type": "root",
+              "name": "ets"
+            },
+            {
+              "type": "root",
+              "name": "@timestamp"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "actor_type",
+              "expr": "$.actor.type"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "context_sid",
+              "expr": "$.context.sid"
+            },
+            {
+              "type": "path",
+              "name": "context_did",
+              "expr": "$.context.did"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_type",
+              "expr": "$.context.cdata[*].type"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_id",
+              "expr": "$.context.cdata[*].id"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l1",
+              "expr": "$.context.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l2",
+              "expr": "$.context.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l3",
+              "expr": "$.context.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l4",
+              "expr": "$.context.rollup.l4"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l3",
+              "expr": "$.object.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l4",
+              "expr": "$.object.rollup.l4"
+            },
+            {
+              "type": "root",
+              "name": "tags"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_subtype",
+              "expr": "$.edata.subtype"
+            },
+            {
+              "type": "path",
+              "name": "edata_mode",
+              "expr": "$.edata.mode"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageid",
+              "expr": "$.edata.pageid"
+            },
+            {
+              "type": "path",
+              "name": "edata_uri",
+              "expr": "$.edata.uri"
+            },
+            {
+              "type": "path",
+              "name": "edata_id",
+              "expr": "$.edata.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "edata_index",
+              "expr": "$.edata.index"
+            },
+            {
+              "type": "path",
+              "name": "edata_pass",
+              "expr": "$.edata.pass"
+            },
+            {
+              "type": "path",
+              "name": "edata_score",
+              "expr": "$.edata.score"
+            },
+            {
+              "type": "path",
+              "name": "edata_resvalues",
+              "expr": "$.edata.resvalues[*]"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_id",
+              "expr": "$.edata.item.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_type",
+              "expr": "$.edata.item.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_title",
+              "expr": "$.edata.item.title"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_maxscore",
+              "expr": "$.edata.item.maxscore"
+            },
+            {
+              "type": "path",
+              "name": "edata_target_id",
+              "expr": "$.edata.target.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_target_type",
+              "expr": "$.edata.target.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_rating",
+              "expr": "$.edata.rating"
+            },
+            {
+              "type": "path",
+              "name": "edata_comments",
+              "expr": "$.edata.comments"
+            },
+            {
+              "type": "path",
+              "name": "edata_dir",
+              "expr": "$.edata.dir"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_id",
+              "expr": "$.edata.items[*].id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_type",
+              "expr": "$.edata.items[*].type"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_origin_id",
+              "expr": "$.edata.items[*].origin.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_origin_type",
+              "expr": "$.edata.items[*].origin.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_to_id",
+              "expr": "$.edata.items[*].to.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_items_to_type",
+              "expr": "$.edata.items[*].to.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_id",
+              "expr": "$.edata.plugin.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_ver",
+              "expr": "$.edata.plugin.ver"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_category",
+              "expr": "$.edata.plugin.category"
+            },
+            {
+              "type": "path",
+              "name": "edata_props",
+              "expr": "$.edata.props[*]"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_prevstate",
+              "expr": "$.edata.prevstate"
+            },
+            {
+              "type": "path",
+              "name": "edata_size",
+              "expr": "$.edata.size"
+            },
+            {
+              "type": "path",
+              "name": "edata_filters_dialcodes",
+              "expr": "$.edata.filters.dialcodes"
+            },
+            {
+              "type": "path",
+              "name": "edata_topn_identifier",
+              "expr": "$.edata.topn[*].identifier"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objid",
+              "expr": "$.edata.visits[*].objid"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objtype",
+              "expr": "$.edata.visits[*].objtype"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_objver",
+              "expr": "$.edata.visits[*].objver"
+            },
+            {
+              "type": "path",
+              "name": "edata_visits_index",
+              "expr": "$.edata.visits[*].index"
+            },
+            {
+              "type": "path",
+              "name": "edata_timespent",
+              "expr": "$.edata.timespent"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageviews",
+              "expr": "$.edata.pageviews"
+            },
+            {
+              "type": "path",
+              "name": "edata_interactions",
+              "expr": "$.edata.interactions"
+            },
+            {
+              "type": "path",
+              "name": "edata_starttime",
+              "expr": "$.edata.starttime"
+            },
+            {
+              "type": "path",
+              "name": "edata_endtime",
+              "expr": "$.edata.endtime"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state",
+              "expr": "$.devicedata.state"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_code",
+              "expr": "$.devicedata.statecode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_city",
+              "expr": "$.devicedata.city"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country_code",
+              "expr": "$.devicedata.countrycode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country",
+              "expr": "$.devicedata.country"
+            },
+            {
+              "type": "path",
+              "name": "device_os",
+              "expr": "$.devicedata.devicespec.os"
+            },
+            {
+              "type": "path",
+              "name": "device_make",
+              "expr": "$.devicedata.devicespec.make"
+            },
+            {
+              "type": "path",
+              "name": "device_id",
+              "expr": "$.devicedata.devicespec.id"
+            },
+            {
+              "type": "path",
+              "name": "device_mem",
+              "expr": "$.devicedata.devicespec.mem"
+            },
+            {
+              "type": "path",
+              "name": "device_idisk",
+              "expr": "$.devicedata.devicespec.idisk"
+            },
+            {
+              "type": "path",
+              "name": "device_edisk",
+              "expr": "$.devicedata.devicespec.edisk"
+            },
+            {
+              "type": "path",
+              "name": "device_scrn",
+              "expr": "$.devicedata.devicespec.scrn"
+            },
+            {
+              "type": "path",
+              "name": "device_camera",
+              "expr": "$.devicedata.devicespec.camera"
+            },
+            {
+              "type": "path",
+              "name": "device_cpu",
+              "expr": "$.devicedata.devicespec.cpu"
+            },
+            {
+              "type": "path",
+              "name": "device_sims",
+              "expr": "$.devicedata.devicespec.sims"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_agent",
+              "expr": "$.devicedata.uaspec.agent"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_ver",
+              "expr": "$.devicedata.uaspec.ver"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_system",
+              "expr": "$.devicedata.uaspec.system"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_platform",
+              "expr": "$.devicedata.uaspec.platform"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_raw",
+              "expr": "$.devicedata.uaspec.raw"
+            },
+            {
+              "type": "path",
+              "name": "device_first_access",
+              "expr": "$.devicedata.firstaccess"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_code",
+              "expr": "$.devicedata.statecustomcode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_name",
+              "expr": "$.devicedata.statecustomname"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_district",
+              "expr": "$.devicedata.districtcustom"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_state",
+              "expr": "$.devicedata.userdeclared.state"
+            },
+            {
+              "type": "path",
+              "name": "user_declared_district",
+              "expr": "$.devicedata.userdeclared.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_channel",
+              "expr": "$.contentdata.channel"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_media_type",
+              "expr": "$.contentdata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_version",
+              "expr": "$.contentdata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "content_last_submitted_on",
+              "expr": "$.contentdata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_published_on",
+              "expr": "$.contentdata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_updated_on",
+              "expr": "$.contentdata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "content_created_by",
+              "expr": "$.contentdata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_channel",
+              "expr": "$.collectiondata.channel"
+            },
+            {
+              "type": "path",
+              "name": "collection_object_type",
+              "expr": "$.collectiondata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_media_type",
+              "expr": "$.collectiondata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "collection_language",
+              "expr": "$.collectiondata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subjects",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_mimetype",
+              "expr": "$.collectiondata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "collection_framework",
+              "expr": "$.collectiondata.framework"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_status",
+              "expr": "$.collectiondata.status"
+            },
+            {
+              "type": "path",
+              "name": "collection_version",
+              "expr": "$.collectiondata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_submitted_on",
+              "expr": "$.collectiondata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_published_on",
+              "expr": "$.collectiondata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_updated_on",
+              "expr": "$.collectiondata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_by",
+              "expr": "$.collectiondata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_grade_list",
+              "expr": "$.userdata.gradelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_language_list",
+              "expr": "$.userdata.languagelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_subject_list",
+              "expr": "$.userdata.subjectlist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_roles",
+              "expr": "$.userdata.roles[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_state",
+              "expr": "$.userdata.state"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_district",
+              "expr": "$.userdata.district"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_block",
+              "expr": "$.userdata.block"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_cluster",
+              "expr": "$.userdata.cluster"
+            },
+            {
+              "type": "path",
+              "name": "user_school_name",
+              "expr": "$.userdata.schoolname"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_batchcode",
+              "expr": "$.dialcodedata.batchcode"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_publisher",
+              "expr": "$.dialcodedata.publisher"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_generated_on",
+              "expr": "$.dialcodedata.generatedon"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_published_on",
+              "expr": "$.dialcodedata.publishedon"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_object_type",
+              "expr": "$.dialcodedata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_iso_state_code",
+              "expr": "$.devicedata.iso3166statecode"
+            },
+            {
+              "type": "path",
+              "name": "content_play_progress",
+              "expr": "$.edata.summary[*].progress"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_created_for",
+              "expr": "$.l2data.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_framework",
+              "expr": "$.l2data.framework"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_name",
+              "expr": "$.l2data.name"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_channel",
+              "expr": "$.l2data.channel"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_mimetype",
+              "expr": "$.l2data.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_medium",
+              "expr": "$.l2data.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_board",
+              "expr": "$.l2data.board"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_contenttype",
+              "expr": "$.l2data.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_gradelevel",
+              "expr": "$.l2data.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_subjects",
+              "expr": "$.l2data.subject[*]"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "mid"
+            },
+            {
+              "type": "long",
+              "name": "ets"
+            },
+            {
+              "type": "string",
+              "name": "@timestamp"
+            },
+            {
+              "type": "string",
+              "name": "actor_id"
+            },
+            {
+              "type": "string",
+              "name": "actor_type"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "context_sid"
+            },
+            {
+              "type": "string",
+              "name": "context_did"
+            },
+            {
+              "name": "context_cdata_type"
+            },
+            {
+              "name": "context_cdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l4"
+            },
+            {
+            "name" : "content_play_progress"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l4"
+            },
+            {
+              "name": "tags"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_subtype"
+            },
+            {
+              "type": "string",
+              "name": "edata_mode"
+            },
+            {
+              "type": "string",
+              "name": "edata_pageid"
+            },
+            {
+              "type": "string",
+              "name": "edata_uri"
+            },
+            {
+              "type": "string",
+              "name": "edata_id"
+            },
+            {
+              "type": "double",
+              "name": "edata_duration"
+            },
+            {
+              "type": "long",
+              "name": "edata_index"
+            },
+            {
+              "type": "string",
+              "name": "edata_pass"
+            },
+            {
+              "type": "double",
+              "name": "edata_score"
+            },
+            {
+              "name": "edata_resvalues"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_title"
+            },
+            {
+              "type": "double",
+              "name": "edata_item_maxscore"
+            },
+            {
+              "type": "string",
+              "name": "edata_target_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_target_type"
+            },
+            {
+              "type": "long",
+              "name": "edata_rating"
+            },
+            {
+              "type": "string",
+              "name": "edata_comments"
+            },
+            {
+              "type": "string",
+              "name": "edata_dir"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_origin_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_origin_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_to_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_items_to_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_ver"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_category"
+            },
+            {
+              "name": "edata_props"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "string",
+              "name": "edata_prevstate"
+            },
+            {
+              "type": "long",
+              "name": "edata_size"
+            },
+            {
+              "name": "edata_filters_dialcodes"
+            },
+            {
+              "type": "string",
+              "name": "edata_topn_identifier"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objid"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objtype"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_objver"
+            },
+            {
+              "type": "string",
+              "name": "edata_visits_index"
+            },
+            {
+              "type": "double",
+              "name": "edata_timespent"
+            },
+            {
+              "type": "long",
+              "name": "edata_pageviews"
+            },
+            {
+              "type": "long",
+              "name": "edata_interactions"
+            },
+            {
+              "type": "long",
+              "name": "edata_starttime"
+            },
+            {
+              "type": "long",
+              "name": "edata_endtime"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_city"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country"
+            },
+            {
+              "type": "string",
+              "name": "device_os"
+            },
+            {
+              "type": "string",
+              "name": "device_make"
+            },
+            {
+              "type": "string",
+              "name": "device_id"
+            },
+            {
+              "type": "long",
+              "name": "device_mem"
+            },
+            {
+              "type": "string",
+              "name": "device_idisk"
+            },
+            {
+              "type": "string",
+              "name": "device_edisk"
+            },
+            {
+              "type": "string",
+              "name": "device_scrn"
+            },
+            {
+              "type": "string",
+              "name": "device_camera"
+            },
+            {
+              "type": "string",
+              "name": "device_cpu"
+            },
+            {
+              "type": "long",
+              "name": "device_sims"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_agent"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_ver"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_system"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_platform"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_raw"
+            },
+            {
+              "type": "long",
+              "name": "device_first_access"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_name"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_state"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "type": "string",
+              "name": "content_media_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_channel"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "double",
+              "name": "content_version"
+            },
+            {
+              "type": "long",
+              "name": "content_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "content_created_by"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_object_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_media_type"
+            },
+            {
+              "name": "collection_language"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_channel"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subjects"
+            },
+            {
+              "type": "string",
+              "name": "collection_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "collection_framework"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_status"
+            },
+            {
+              "type": "double",
+              "name": "collection_version"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_by"
+            },
+            {
+              "name": "collection_created_for"
+            },
+            {
+              "name": "user_grade_list"
+            },
+            {
+              "name": "user_language_list"
+            },
+            {
+              "name": "user_subject_list"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "name": "user_roles"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_block"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_cluster"
+            },
+            {
+              "type": "string",
+              "name": "user_school_name"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_batchcode"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_publisher"
+            },
+            {
+              "type": "long",
+              "name": "dialcode_generated_on"
+            },
+            {
+              "type": "long",
+              "name": "dialcode_published_on"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_object_type"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_iso_state_code"
+            },
+            {
+              "name": "object_l2_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_framework"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_name"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_channel"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_mimetype"
+            },
+            {
+              "name": "object_l2_medium"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_board"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_contenttype"
+            },
+            {
+              "name": "object_l2_gradelevel"
+            },
+            {
+              "name": "object_l2_subjects"
+            },
+            {
+              "type": "string",
+              "name": "utm_source"
+            },
+            {
+              "type": "string",
+              "name": "utm_content"
+            },
+            {
+              "type": "string",
+              "name": "utm_medium"
+            },
+            {
+              "type": "string",
+              "name": "utm_campaign"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "day",
+      "rollup": false
+    },
+    "transformSpec": {
+      "transforms": [
+        {
+          "type": "expression",
+          "name": "utm_source",
+          "expression": "if(like(\"context_cdata_type\",'UtmSource'), context_cdata_id, '')"
+        },
+        {
+          "type": "expression",
+          "name": "utm_content",
+          "expression": "if(like(\"context_cdata_type\",'UtmContent'), context_cdata_id, '')"
+        },
+        {
+          "type": "expression",
+          "name": "utm_medium",
+          "expression": "if(like(\"context_cdata_type\",'UtmMedium'), context_cdata_id, '')"
+        },
+        {
+          "type": "expression",
+          "name": "utm_campaign",
+          "expression": "if(like(\"context_cdata_type\",'UtmCampaign'), context_cdata_id, '')"
+        }
+      ]
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{raw_telemetry_events_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT7200S",
+    "useEarliestOffset": true,
+    "completionTimeout" : "PT7200S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 2000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events b/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events
new file mode 100644
index 0000000000..2793650757
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/raw_telemetry_feedback_events
@@ -0,0 +1,864 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "telemetry-feedback-events",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "root",
+              "name": "mid"
+            },
+            {
+              "type": "root",
+              "name": "ets"
+            },
+            {
+              "type": "root",
+              "name": "@timestamp"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "actor_type",
+              "expr": "$.actor.type"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "context_sid",
+              "expr": "$.context.sid"
+            },
+            {
+              "type": "path",
+              "name": "context_did",
+              "expr": "$.context.did"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_type",
+              "expr": "$.context.cdata[*].type"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_id",
+              "expr": "$.context.cdata[*].id"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l1",
+              "expr": "$.context.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l2",
+              "expr": "$.context.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l3",
+              "expr": "$.context.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "context_rollup_l4",
+              "expr": "$.context.rollup.l4"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l3",
+              "expr": "$.object.rollup.l3"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l4",
+              "expr": "$.object.rollup.l4"
+            },
+            {
+              "type": "root",
+              "name": "tags"
+            },
+            {
+              "type": "path",
+              "name": "edata_rating",
+              "expr": "$.edata.rating"
+            },
+            {
+              "type": "path",
+              "name": "edata_comments",
+              "expr": "$.edata.comments"
+            },
+            {
+              "type": "path",
+              "name": "edata_commentid",
+              "expr": "$.edata.commentid"
+            },
+            {
+              "type": "path",
+              "name": "edata_commenttxt",
+              "expr": "$.edata.commenttxt"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state",
+              "expr": "$.devicedata.state"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_code",
+              "expr": "$.devicedata.statecode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_city",
+              "expr": "$.devicedata.city"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country_code",
+              "expr": "$.devicedata.countrycode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_country",
+              "expr": "$.devicedata.country"
+            },
+            {
+              "type": "path",
+              "name": "device_os",
+              "expr": "$.devicedata.devicespec.os"
+            },
+            {
+              "type": "path",
+              "name": "device_make",
+              "expr": "$.devicedata.devicespec.make"
+            },
+            {
+              "type": "path",
+              "name": "device_id",
+              "expr": "$.devicedata.devicespec.id"
+            },
+            {
+              "type": "path",
+              "name": "device_mem",
+              "expr": "$.devicedata.devicespec.mem"
+            },
+            {
+              "type": "path",
+              "name": "device_idisk",
+              "expr": "$.devicedata.devicespec.idisk"
+            },
+            {
+              "type": "path",
+              "name": "device_edisk",
+              "expr": "$.devicedata.devicespec.edisk"
+            },
+            {
+              "type": "path",
+              "name": "device_scrn",
+              "expr": "$.devicedata.devicespec.scrn"
+            },
+            {
+              "type": "path",
+              "name": "device_camera",
+              "expr": "$.devicedata.devicespec.camera"
+            },
+            {
+              "type": "path",
+              "name": "device_cpu",
+              "expr": "$.devicedata.devicespec.cpu"
+            },
+            {
+              "type": "path",
+              "name": "device_sims",
+              "expr": "$.devicedata.devicespec.sims"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_agent",
+              "expr": "$.devicedata.uaspec.agent"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_ver",
+              "expr": "$.devicedata.uaspec.ver"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_system",
+              "expr": "$.devicedata.uaspec.system"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_platform",
+              "expr": "$.devicedata.uaspec.platform"
+            },
+            {
+              "type": "path",
+              "name": "device_uaspec_raw",
+              "expr": "$.devicedata.uaspec.raw"
+            },
+            {
+              "type": "path",
+              "name": "device_first_access",
+              "expr": "$.devicedata.firstaccess"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_code",
+              "expr": "$.devicedata.statecustomcode"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_state_custom_name",
+              "expr": "$.devicedata.statecustomname"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_district",
+              "expr": "$.devicedata.districtcustom"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_media_type",
+              "expr": "$.contentdata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_version",
+              "expr": "$.contentdata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "content_last_submitted_on",
+              "expr": "$.contentdata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_published_on",
+              "expr": "$.contentdata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_updated_on",
+              "expr": "$.contentdata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "user_grade_list",
+              "expr": "$.userdata.gradelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_language_list",
+              "expr": "$.userdata.languagelist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_subject_list",
+              "expr": "$.userdata.subjectlist[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_roles",
+              "expr": "$.userdata.roles[*]"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_state",
+              "expr": "$.userdata.state"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_district",
+              "expr": "$.userdata.district"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "device_loc_iso_state_code",
+              "expr": "$.devicedata.iso3166statecode"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_created_for",
+              "expr": "$.l2data.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_framework",
+              "expr": "$.l2data.framework"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_name",
+              "expr": "$.l2data.name"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_channel",
+              "expr": "$.l2data.channel"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_mimetype",
+              "expr": "$.l2data.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_medium",
+              "expr": "$.l2data.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_board",
+              "expr": "$.l2data.board"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_contenttype",
+              "expr": "$.l2data.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_gradelevel",
+              "expr": "$.l2data.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_subjects",
+              "expr": "$.l2data.subject[*]"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "mid"
+            },
+            {
+              "type": "long",
+              "name": "ets"
+            },
+            {
+              "type": "string",
+              "name": "@timestamp"
+            },
+            {
+              "type": "string",
+              "name": "actor_id"
+            },
+            {
+              "type": "string",
+              "name": "actor_type"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "context_sid"
+            },
+            {
+              "type": "string",
+              "name": "context_did"
+            },
+            {
+              "name": "context_cdata_type"
+            },
+            {
+              "name": "context_cdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "context_rollup_l4"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l3"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l4"
+            },
+            {
+              "name": "tags"
+            },
+            {
+              "type": "long",
+              "name": "edata_rating"
+            },
+            {
+              "type": "string",
+              "name": "edata_comments"
+            },
+            {
+              "type": "string",
+              "name": "edata_commentid"
+            },
+            {
+              "type": "string",
+              "name": "edata_commenttxt"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_city"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_country"
+            },
+            {
+              "type": "string",
+              "name": "device_os"
+            },
+            {
+              "type": "string",
+              "name": "device_make"
+            },
+            {
+              "type": "string",
+              "name": "device_id"
+            },
+            {
+              "type": "long",
+              "name": "device_mem"
+            },
+            {
+              "type": "string",
+              "name": "device_idisk"
+            },
+            {
+              "type": "string",
+              "name": "device_edisk"
+            },
+            {
+              "type": "string",
+              "name": "device_scrn"
+            },
+            {
+              "type": "string",
+              "name": "device_camera"
+            },
+            {
+              "type": "string",
+              "name": "device_cpu"
+            },
+            {
+              "type": "long",
+              "name": "device_sims"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_agent"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_ver"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_system"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_platform"
+            },
+            {
+              "type": "string",
+              "name": "device_uaspec_raw"
+            },
+            {
+              "type": "long",
+              "name": "device_first_access"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_code"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_state_custom_name"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "type": "string",
+              "name": "content_media_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "double",
+              "name": "content_version"
+            },
+            {
+              "type": "long",
+              "name": "content_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_updated_on"
+            },
+            {
+              "name": "user_grade_list"
+            },
+            {
+              "name": "user_language_list"
+            },
+            {
+              "name": "user_subject_list"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "name": "user_roles"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_iso_state_code"
+            },
+            {
+              "name": "object_l2_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_framework"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_name"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_channel"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_mimetype"
+            },
+            {
+              "name": "object_l2_medium"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_board"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_contenttype"
+            },
+            {
+              "name": "object_l2_gradelevel"
+            },
+            {
+              "name": "object_l2_subjects"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "count"
+      },
+      {
+        "type": "longSum",
+        "name": "total_rating",
+        "fieldName": "edata_rating"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "day",
+      "rollup": true
+    },
+    "transformSpec": {
+      "filter": {
+        "type": "selector",
+        "dimension": "eid",
+        "value": "FEEDBACK"
+      }
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{raw_telemetry_feedback_events_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true,
+    "completionTimeout": "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts b/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts
new file mode 100644
index 0000000000..8246184b9d
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_course_completion_audit_syncts
@@ -0,0 +1,605 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "course-completion-events-rollup",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "actor_type",
+              "expr": "$.actor.type"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_prevstate",
+              "expr": "$.edata.prevstate"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_created_by",
+              "expr": "$.contentdata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_object_type",
+              "expr": "$.collectiondata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_media_type",
+              "expr": "$.collectiondata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "collection_language",
+              "expr": "$.collectiondata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subjects",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_mimetype",
+              "expr": "$.collectiondata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "collection_framework",
+              "expr": "$.collectiondata.framework"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_status",
+              "expr": "$.collectiondata.status"
+            },
+            {
+              "type": "path",
+              "name": "collection_version",
+              "expr": "$.collectiondata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_by",
+              "expr": "$.collectiondata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_created_for",
+              "expr": "$.l2data.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_framework",
+              "expr": "$.l2data.framework"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_name",
+              "expr": "$.l2data.name"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_channel",
+              "expr": "$.l2data.channel"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_mimetype",
+              "expr": "$.l2data.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_medium",
+              "expr": "$.l2data.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_board",
+              "expr": "$.l2data.board"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_contenttype",
+              "expr": "$.l2data.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_gradelevel",
+              "expr": "$.l2data.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_subjects",
+              "expr": "$.l2data.subject[*]"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "actor_id"
+            },
+            {
+              "type": "string",
+              "name": "actor_type"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "string",
+              "name": "edata_prevstate"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "string",
+              "name": "content_created_by"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_object_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "name": "collection_language"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subjects"
+            },
+            {
+              "type": "string",
+              "name": "collection_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "collection_framework"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_status"
+            },
+            {
+              "type": "double",
+              "name": "collection_version"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_by"
+            },
+            {
+              "name": "collection_created_for"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "name": "object_l2_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_framework"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_name"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_channel"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_mimetype"
+            },
+            {
+              "name": "object_l2_medium"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_board"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_contenttype"
+            },
+            {
+              "name": "object_l2_gradelevel"
+            },
+            {
+              "name": "object_l2_subjects"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "transformSpec": {
+      "filter": {
+        "type": "and",
+        "fields": [
+          {
+            "type": "selector",
+            "dimension": "eid",
+            "value": "AUDIT"
+          },
+          {
+            "type": "selector",
+            "dimension": "edata_type",
+            "value": "enrol-complete"
+          }
+        ]
+      }
+    },
+    "metricsSpec": [
+      {
+        "name": "total_count",
+        "type": "count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_duration",
+        "fieldName": "edata_duration"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "rollup": true
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{rollup_course_completion_audit_syncts_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": false,
+    "completionTimeout": "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+     "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts
new file mode 100644
index 0000000000..738bb5dd90
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_error_hourly_syncts
@@ -0,0 +1,315 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "error-hourly-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subject",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subject",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "edata_err",
+              "expr": "$.edata.err"
+            },
+            {
+              "type": "path",
+              "name": "edata_errtype",
+              "expr": "$.edata.errtype"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_id",
+              "expr": "$.edata.plugin.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_plugin_ver",
+              "expr": "$.edata.plugin.ver"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subject"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subject"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_err"
+            },
+            {
+              "type": "string",
+              "name": "edata_errtype"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_plugin_ver"
+            }
+          ]
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "total_count"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "HOUR",
+      "rollup": true
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.error",
+    "replicas": 1,
+    "taskCount": {{rollup_error_hourly_syncts_taskcount}},
+    "taskDuration": "PT3600S",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "useEarliestOffset": true,
+    "completionTimeout": "PT1800S"
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_ml_observation_status b/ansible/roles/druid-ingestion/templates/rollup_ml_observation_status
new file mode 100644
index 0000000000..5218c0f298
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_ml_observation_status
@@ -0,0 +1,266 @@
+{
+  "type": "kafka",
+  "spec": {
+    "dataSchema": {
+      "dataSource": "ml-observation-status-rollup",
+      "timestampSpec": {
+        "column": "updatedAt",
+        "format": "iso"
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          "solution_id",
+          "submission_id",
+          "entity_name",
+          "solution_name",
+          "role_title",
+          "state_externalId",
+          "block_externalId",
+          "district_externalId",
+          "cluster_externalId",
+          "school_externalId",
+          "state_name",
+          "block_name",
+          "district_name",
+          "cluster_name",
+          "school_name",
+          "board_name",
+          "completedDate",
+          "channel",
+          "program_id",
+          "program_name",
+          "private_program",
+          "organisation_id",
+          "organisation_name",
+          "user_id",
+          "solution_type",
+          "status",
+          "parent_channel",
+          "updatedAt",
+          {
+            "type": "long",
+            "name": "status_code"
+          }
+        ]
+      },
+      "metricsSpec": [
+        {
+          "type": "count",
+          "name": "count"
+        },
+        {
+          "type": "longSum",
+          "name": "sum___v",
+          "fieldName": "__v",
+          "expression": null
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_distinct_solution",
+          "fieldName": "solution_id",
+          "lgK": 12,
+          "tgtHllType": "HLL_4",
+          "round": false
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_distinct_submission_id",
+          "fieldName": "submission_id",
+          "lgK": 12,
+          "tgtHllType": "HLL_4",
+          "round": false
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_distinct_user_id",
+          "fieldName": "user_id",
+          "lgK": 12,
+          "tgtHllType": "HLL_4",
+          "round": false
+        }
+      ],
+      "granularitySpec": {
+        "type": "uniform",
+        "segmentGranularity": "DAY",
+        "queryGranularity": "DAY",
+        "rollup": true
+      },
+      "transformSpec": {
+        "filter": null,
+        "transforms": [
+          {
+            "type": "expression",
+            "name": "parent_channel",
+            "expression": "nvl(\"parent_channel\", 'SHIKSHALOKAM')"
+          },
+          {
+            "type": "expression",
+            "name": "status_code",
+            "expression": "case_simple(\"status\",'started',1,'inprogress',3,'ratingPending',5,'completed',7,0)"
+          }
+        ]
+      }
+    },
+    "ioConfig": {
+      "topic": "{{env}}.ml.survey.raw",
+      "taskCount":"{{rollup_ml_observation_status_taskcount}}",
+      "replicas":1,
+      "taskDuration":"PT14400S",
+      "completionTimeout":"PT1800S",
+      "inputFormat": {
+        "type": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": true,
+          "fields": [
+            {
+              "type": "jq",
+              "name": "solution_id",
+              "expr": ".solutionId"
+            },
+            {
+              "type": "jq",
+              "name": "submission_id",
+              "expr": "._id"
+            },
+            {
+              "type": "jq",
+              "name": "entity_name",
+              "expr": ".entity_name"
+            },
+            {
+              "type": "jq",
+              "name": "solution_name",
+              "expr": ".solutionInfo ? | .name"
+            },
+            {
+              "type": "jq",
+              "name": "role_title",
+              "expr": ".role_title"
+            },
+            {
+              "type": "jq",
+              "name": "state_externalId",
+              "expr": ".userRoleInformation? | .state"
+            },
+            {
+              "type": "jq",
+              "name": "block_externalId",
+              "expr": ".userRoleInformation? | .block"
+            },
+            {
+              "type": "jq",
+              "name": "district_externalId",
+              "expr": ".userRoleInformation? | .district"
+            },
+            {
+              "type": "jq",
+              "name": "cluster_externalId",
+              "expr": ".userRoleInformation? | .cluster"
+            },
+            {
+              "type": "jq",
+              "name": "school_externalId",
+              "expr": ".userRoleInformation? | .school"
+            },
+            {
+              "type": "jq",
+              "name": "state_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"state\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"state\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "block_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"block\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"block\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "district_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"district\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"district\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "cluster_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "school_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"school\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"school\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "board_name",
+              "expr": "if .userProfile? | .framework ? | .board | length > 0 then [.userProfile? | .framework ? | .board[]] | join(\",\") else null end"
+            },
+            {
+              "type": "jq",
+              "name": "completedDate",
+              "expr": ".completedDate"
+            },
+            {
+              "type": "jq",
+              "name": "channel",
+              "expr": ".userProfile ? | .rootOrgId"
+            },
+            {
+              "type": "jq",
+              "name": "program_id",
+              "expr": "if .programId then .programId else null end"
+            },
+            {
+              "type": "jq",
+              "name": "program_name",
+              "expr": ".programInformation? | .name"
+            },
+            {
+              "type": "jq",
+              "name": "private_program",
+              "expr": "if .isAPrivateProgram == true then \"true\" elif .isAPrivateProgram == false then \"false\" else \"true\" end"
+            },
+            {
+              "type": "jq",
+              "name": "organisation_id",
+              "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .organisationId else null end"
+            },
+            {
+              "type": "jq",
+              "name": "organisation_name",
+              "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .orgName else null end"
+            },
+            {
+              "type": "jq",
+              "name": "user_id",
+              "expr": ".createdBy"
+            },
+            {
+              "type": "jq",
+              "name": "solution_type",
+              "expr": "if (.isRubricDriven ? and .criteriaLevelReport ?) then \"observation_with_rubric\" elif (.isRubricDriven ? | not) then  \"observation_without_rubric\" elif (.isRubricDriven ? and (.criteriaLevelReport ? | not)) then  \"observation_without_rubric\"  else \"observation_without_rubric\" end"
+            },
+            {
+              "type": "jq",
+              "name": "updatedAt",
+              "expr": ".updatedAt"
+            },
+            {
+              "type": "jq",
+              "name": "status",
+              "expr": ".status"
+            }
+          ]
+        },
+        "featureSpec": {}
+      },
+      "consumerProperties": {
+        "bootstrap.servers": "{{kafka_brokers}}"
+      },
+      "useEarliestOffset": true,
+      "type": "kafka"
+    },
+    "tuningConfig": {
+      "type": "kafka",
+      "reportParseExceptions":false,
+      "maxRowsPerSegment":5000000
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_ml_project_status b/ansible/roles/druid-ingestion/templates/rollup_ml_project_status
new file mode 100644
index 0000000000..256315cd43
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_ml_project_status
@@ -0,0 +1,291 @@
+{
+    "type": "kafka",
+    "spec": {
+      "ioConfig": {
+        "type": "kafka",
+        "completionTimeout": "PT1800S",
+        "consumerProperties": {
+          "bootstrap.servers": "{{kafka_brokers}}"
+        },
+        "replicas": 1,
+        "taskCount": "{{rollup_ml_project_status_taskcount}}",
+        "topic": "{{env}}.ml.projects.submissions",
+        "taskDuration":"PT14400S",
+        "inputFormat": {
+          "type": "json",
+          "flattenSpec": {
+            "fields": [
+              {
+                "type": "jq",
+                "name": "project_id",
+                "expr": "._id"
+              },
+              {
+                "type": "jq",
+                "name": "status_of_project",
+                "expr": ".status"
+              },
+              {
+                "type": "jq",
+                "name": "project_title",
+                "expr": ".solutionInformation? | .name"
+              },
+              {
+                "type": "jq",
+                "name": "tasks_name",
+                "expr": "[.tasks[]? | .name]"
+              },
+              {
+                "type": "jq",
+                "name": "tasks_status",
+                "expr": "[.tasks[]? | .status]"
+              },
+              {
+                "expr": ".userRoleInformation? | .role",
+                "name": "designation",
+                "type": "jq"
+              },
+              {
+                "expr": "[.tasks[]? | ._id]",
+                "name": "task_id",
+                "type": "jq"
+              },
+              {
+                "expr": "if .programId then .programId else null end",
+                "name": "program_id",
+                "type": "jq"
+              },
+              {
+                "expr": ".programInformation? | .name",
+                "name": "program_name",
+                "type": "jq"
+              },
+              {
+                "expr": ".updatedAt",
+                "name": "project_updated_date",
+                "type": "jq"
+              },
+              {
+                "expr": ".userId",
+                "name": "createdBy",
+                "type": "jq"
+              },
+              {
+                "expr": "if .programExternalId then .programExternalId else null end",
+                "name": "program_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": "if .isAPrivateProgram == true then \"true\" elif .isAPrivateProgram == false then \"false\" else \"true\" end",
+                "name": "private_program",
+                "type": "jq"
+              },
+              {
+                "expr": "if .hasAcceptedTAndC == true then \"true\" elif .hasAcceptedTAndC == false then \"false\" else \"false\" end",
+                "name": "project_terms_and_condition",
+                "type": "jq"
+              },
+              {
+                "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .organisationId else null end",
+                "name": "organisation_id",
+                "type": "jq"
+              },
+              {
+                "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .orgName else null end",
+                "name": "organisation_name",
+                "type": "jq"
+              },
+              {
+                "expr": ".solutionInformation? | ._id",
+                "name": "solution_id",
+                "type": "jq"
+              },
+              {
+                "expr": ".userRoleInformation? | .state",
+                "name": "state_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": ".userRoleInformation? | .block",
+                "name": "block_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": ".userRoleInformation? | .district",
+                "name": "district_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": ".userRoleInformation? | .cluster",
+                "name": "cluster_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": ".userRoleInformation? | .school",
+                "name": "school_externalId",
+                "type": "jq"
+              },
+              {
+                "expr": "if [.tasks[]? | .attachments[]?| select(.source | . != \"\" and . != null)]|length > 0 then \"true\" else \"false\" end",
+                "name": "task_evidence_status",
+                "type": "jq"
+              },
+              {
+                "expr": "[.categories[]? | .name] | join(\",\")",
+                "name": "area_of_improvement",
+                "type": "jq"
+              },
+              {
+                "expr": "if .projectTemplateId then \"project imported from library\" else \"user created project\" end",
+                "name": "project_created_type",
+                "type": "jq"
+              },
+              {
+                "type": "jq",
+                "name": "state_name",
+                "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"state\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"state\")) | .name else null end"
+              },
+              {
+                "type": "jq",
+                "name": "block_name",
+                "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"block\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"block\")) | .name else null end"
+              },
+              {
+                "type": "jq",
+                "name": "district_name",
+                "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"district\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"district\")) | .name else null end"
+              },
+              {
+                "type": "jq",
+                "name": "cluster_name",
+                "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\")) | .name else null end"
+              },
+              {
+                "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"school\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"school\")) | .name else null end",
+                "name": "school_name",
+                "type": "jq"
+              },
+              {
+                "expr": "if .userProfile? | .framework ? | .board | length > 0 then [.userProfile? | .framework ? | .board[]] | join(\",\") else null end",
+                "name": "board_name",
+                "type": "jq"
+              },
+              {
+                "expr": "if ([[.tasks[]? |.attachments[]?]|length] + [.attachments?|length])|add > 0 then \"true\" else \"false\" end",
+                "name": "evidence_status",
+                "type": "jq"
+              }
+            ]
+          }
+        },
+        "useEarliestOffset": true
+      },
+      "tuningConfig": {
+        "type": "kafka",
+        "maxRowsPerSegment": 5000000,
+        "reportParseExceptions": false
+      },
+      "dataSchema": {
+        "dataSource": "ml-project-status-rollup",
+        "timestampSpec": {
+          "column": "project_updated_date",
+          "format": "iso"
+        },
+        "transformSpec": {
+          "transforms": [
+            {
+              "type": "expression",
+              "name": "parent_channel",
+              "expression": "nvl(\"parent_channel\", 'SHIKSHALOKAM')"
+            },
+            {
+              "type": "expression",
+              "name": "status_code",
+              "expression": "case_simple(\"status_of_project\",'started',1,'inProgress',3,'submitted',5,0)"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            "project_id",
+            "status_of_project",
+            "project_title",
+            "tasks_name",
+            "tasks_status",
+            "designation",
+            "task_id",
+            "program_id",
+            "program_name",
+            "createdBy",
+            "program_externalId",
+            "private_program",
+            "project_terms_and_condition",
+            "organisation_id",
+            "organisation_name",
+            "solution_id",
+            "state_externalId",
+            "block_externalId",
+            "district_externalId",
+            "cluster_externalId",
+            "school_externalId",
+            "task_evidence_status",
+            "area_of_improvement",
+            "project_created_type",
+            "state_name",
+            "block_name",
+            "district_name",
+            "cluster_name",
+            "school_name",
+            "board_name",
+            "parent_channel",
+            {
+              "type": "long",
+              "name": "status_code"
+            }
+          ]
+        },
+        "granularitySpec": {
+          "queryGranularity": "day",
+          "rollup": true,
+          "segmentGranularity": "day",
+          "type": "uniform"
+        },
+        "metricsSpec": [
+          {
+            "name": "count",
+            "type": "count"
+          },
+          {
+            "name": "sum___v",
+            "type": "longSum",
+            "fieldName": "__v"
+          },
+          {
+            "name": "sum_status_code",
+            "type": "longMax",
+            "fieldName": "status_code"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_createdBy",
+            "fieldName": "createdBy"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_project_id",
+            "fieldName": "project_id"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_solution_id",
+            "fieldName": "solution_id"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_program_id",
+            "fieldName": "program_id"
+          }
+        ]
+      }
+    }
+  }
diff --git a/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status b/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status
new file mode 100644
index 0000000000..33aec75483
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_ml_survey_status
@@ -0,0 +1,247 @@
+{
+  "type": "kafka",
+  "spec": {
+    "ioConfig": {
+      "type": "kafka",
+      "consumerProperties": {
+        "bootstrap.servers": "{{kafka_brokers}}"
+      },      
+      "topic": "{{env}}.ml.survey.raw",
+      "taskCount":"{{rollup_ml_survey_status_taskcount}}",
+      "replicas":1,
+      "taskDuration":"PT14400S",
+      "completionTimeout":"PT1800S",
+      "inputFormat": {
+        "type": "json",
+        "flattenSpec": {
+          "fields": [
+            {
+              "type": "jq",
+              "name": "survey_submission_id",
+              "expr": "._id"
+            },
+            {
+              "type": "jq",
+              "name": "submission_status",
+              "expr": ".status"
+            },
+            {
+              "type": "jq",
+              "name": "user_id",
+              "expr": ".createdBy"
+            },
+            {
+              "type": "jq",
+              "name": "user_sub_type",
+              "expr": ".userRoleInformation? | .role"
+            },
+            {
+              "type": "jq",
+              "name": "user_type",
+              "expr": "[.userProfile? | .profileUserTypes[] ?| .type?] | unique | join(\",\")"
+            },
+            {
+              "expr": ".userRoleInformation? | .state",
+              "name": "state_externalId",
+              "type": "jq"
+            },
+            {
+              "expr": ".userRoleInformation? | .block",
+              "name": "block_externalId",
+              "type": "jq"
+            },
+            {
+              "expr": ".userRoleInformation? | .district",
+              "name": "district_externalId",
+              "type": "jq"
+            },
+            {
+              "expr": ".userRoleInformation? | .cluster",
+              "name": "cluster_externalId",
+              "type": "jq"
+            },
+            {
+              "expr": ".userRoleInformation? | .school",
+              "name": "school_externalId",
+              "type": "jq"
+            },
+            {
+              "type": "jq",
+              "name": "state_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"state\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"state\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "block_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"block\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"block\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "district_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"district\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"district\")) | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "cluster_name",
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"cluster\")) | .name else null end"
+            },
+            {
+              "expr": "if [.userProfile ? |.userLocations [] ? | select(.type | contains(\"school\"))] | length > 0 then .userProfile ? |.userLocations [] ? | select(.type | contains(\"school\")) | .name else null end",
+              "name": "school_name",
+              "type": "jq"
+            },
+            {
+              "expr": "if .userProfile? | .framework ? | .board | length > 0 then [.userProfile? | .framework ? | .board[]] | join(\",\") else null end",
+              "name": "board_name",
+              "type": "jq"
+            },
+            {
+              "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .organisationId else null end",
+              "name": "organisation_id",
+              "type": "jq"
+            },
+            {
+              "expr": "if [.userProfile? | .organisations[]? | select (.isSchool == false)] | length > 0 then .userProfile? | .organisations[]? | select(.isSchool == false) | .orgName else null end",
+              "name": "organisation_name",
+              "type": "jq"
+            },
+            {
+              "expr": "if .programExternalId then .programExternalId else null end",
+              "name": "program_externalId",
+              "type": "jq"
+            },
+            {
+              "expr": "if .programId then .programId else null end",
+              "name": "program_id",
+              "type": "jq"
+            },
+            {
+              "expr": "if (.programInformation? | .name) then .programInformation? | .name else null end",
+              "name": "program_name",
+              "type": "jq"
+            },
+            {
+              "expr": "if (.surveyInformation? | .name) then .surveyInformation? | .name else null end",
+              "name": "survey_name",
+              "type": "jq"
+            },
+            {
+              "expr": "if .surveyId then .surveyId else null end",
+              "name": "survey_id",
+              "type": "jq"
+            },
+            {
+              "expr": "if (.surveyInformation? | .solutionExternalId) then .surveyInformation? | .solutionExternalId else null end",
+              "name": "survey_externalId",
+              "type": "jq"
+            },
+            {
+              "type": "jq",
+              "name": "created_date",
+              "expr": ".createdAt"
+            },
+            {
+              "type": "jq",
+              "name": "submission_date",
+              "expr": "if .completedDate then .completedDate else null end"
+            },
+            {
+              "type": "jq",
+              "name": "updatedAt",
+              "expr": "if .updatedAt then .updatedAt else null end"
+            },
+            {
+              "type": "jq",
+              "name": "solution_name",
+              "expr": "if (.solutionInformation? | .name) then .solutionInformation? | .name else null end"
+            },
+            {
+              "type": "jq",
+              "name": "solution_id",
+              "expr": "if .solutionId then .solutionId else null end"
+            },
+            {
+              "type": "jq",
+              "name": "private_program",
+              "expr": "if .isAPrivateProgram == true then \"true\" elif .isAPrivateProgram == false then \"false\" else \"true\" end"
+            }
+          ]
+        }
+      },
+      "useEarliestOffset": true
+    },
+    "tuningConfig": {
+      "type": "kafka",
+      "reportParseExceptions": false,
+      "maxRowsPerSegment": 5000000
+    },
+    "dataSchema": {
+      "dataSource": "ml-survey-status-rollup",
+      "timestampSpec": {
+        "column": "updatedAt",
+        "format": "iso"
+      },
+      "transformSpec": {
+        "transforms": [
+          {
+            "type": "expression",
+            "name": "parent_channel",
+            "expression": "nvl(\"parent_channel\", 'SHIKSHALOKAM')"
+          },
+          {
+            "type": "expression",
+            "name": "status_code",
+            "expression": "case_simple(\"status\",'started',1,'inprogress',3,'ratingPending',5,'completed',7,0)"
+          }
+        ]
+      },
+      "dimensionsSpec": {
+        "dimensions": [
+          "survey_submission_id","submission_status","user_id","user_sub_type","user_type","state_externalId","block_externalId","district_externalId","cluster_externalId","school_externalId","state_name","block_name","district_name","cluster_name","school_name","board_name","organisation_id","organisation_name","program_externalId","program_id","program_name","survey_name","survey_id","survey_externalId","created_date","submission_date","updatedAt","parent_channel",{"type":"long","name":"status_code"},"solution_name","solution_id","private_program"
+        ]
+      },
+      "granularitySpec": {
+        "type": "uniform",
+        "queryGranularity": "day",
+        "rollup": true,
+        "segmentGranularity": "day"
+      },
+      "metricsSpec": [
+        {
+          "name": "count",
+          "type": "count"
+        },
+        {
+          "name": "sum___v",
+          "type": "longSum",
+          "fieldName": "__v"
+        },
+        {
+          "name": "sum_status_code",
+          "type": "longMax",
+          "fieldName": "status_code"
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_of_user_id",
+          "fieldName": "user_id"
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_of_survey_submission_id",
+          "fieldName": "survey_submission_id"
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_of_solution_id",
+          "fieldName": "solution_id"
+        },
+        {
+          "type": "HLLSketchBuild",
+          "name": "count_of_program_id",
+          "fieldName": "program_id"
+        }
+      ]
+    }
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts b/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts
new file mode 100644
index 0000000000..108c77af35
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_summary_distinct_counts
@@ -0,0 +1,375 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "summary-distinct-counts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "syncts"
+            },
+            {
+              "type": "root",
+              "name": "uid"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_id",
+              "expr": "$.dimensions.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_pid",
+              "expr": "$.dimensions.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_type",
+              "expr": "$.dimensions.type"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_ver",
+              "expr": "$.dimensions.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_mode",
+              "expr": "$.dimensions.mode"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_channel",
+              "expr": "$.contentdata.channel"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subject",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_channel",
+              "expr": "$.collectiondata.channel"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subject",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_block",
+              "expr": "$.userdata.block"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_cluster",
+              "expr": "$.userdata.cluster"
+            },
+            {
+              "type": "path",
+              "name": "user_school_name",
+              "expr": "$.userdata.schoolname"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_did",
+              "expr": "$.dimensions.did"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "dimensions_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_type"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_mode"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_channel"
+            },
+            {
+              "name": "content_subject"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_block"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_cluster"
+            },
+            {
+              "type": "string",
+              "name": "user_school_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "name": "collection_channel"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subject"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_for"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_devices",
+        "fieldName": "dimensions_did"
+      },
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_users",
+        "fieldName": "uid"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "day",
+      "queryGranularity": "day",
+      "rollup": false
+    }
+  },
+  "ioConfig": {
+   "topic": "{{env}}.druid.events.summary",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{rollup_summary_distinct_counts_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": true,
+    "completionTimeout": "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_summary_syncts b/ansible/roles/druid-ingestion/templates/rollup_summary_syncts
new file mode 100644
index 0000000000..0c54f6684d
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_summary_syncts
@@ -0,0 +1,384 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "summary-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "syncts"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_id",
+              "expr": "$.dimensions.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_pid",
+              "expr": "$.dimensions.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_type",
+              "expr": "$.dimensions.type"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_pdata_ver",
+              "expr": "$.dimensions.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "dimensions_mode",
+              "expr": "$.dimensions.mode"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_channel",
+              "expr": "$.contentdata.channel"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subject",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_channel",
+              "expr": "$.collectiondata.channel"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subject",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_keywords",
+              "expr": "$.collectiondata.keywords[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_block",
+              "expr": "$.userdata.block"
+            },
+            {
+              "type": "path",
+              "name": "user_loc_cluster",
+              "expr": "$.userdata.cluster"
+            },
+            {
+              "type": "path",
+              "name": "user_school_name",
+              "expr": "$.userdata.schoolname"
+            },
+            {
+              "type": "path",
+              "name": "edata_interaction_count",
+              "expr": "$.edata.eks.interact_events_count"
+            },
+            {
+              "type": "path",
+              "name": "edata_time_spent",
+              "expr": "$.edata.eks.time_spent"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "dimensions_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_type"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "dimensions_mode"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "name": "content_channel"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subject"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_block"
+            },
+            {
+              "type": "string",
+              "name": "user_loc_cluster"
+            },
+            {
+              "type": "string",
+              "name": "user_school_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "name": "collection_channel"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_keywords"
+            },
+            {
+              "name": "collection_subject"
+            },
+            {
+              "name": "collection_created_for"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "total_count"
+      },
+      {
+        "type": "longSum",
+        "name": "total_interactions",
+        "fieldName": "edata_interaction_count",
+        "expression": null
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_time_spent",
+        "fieldName": "edata_time_spent",
+        "expression": null
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "rollup": true,
+      "intervals": null
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 5000000
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.summary",
+    "replicas": 1,
+    "taskCount": {{rollup_summary_syncts_taskcount}},
+    "taskDuration": "PT14400S",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "useEarliestOffset": false,
+    "completionTimeout": "PT1800S"
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts
new file mode 100644
index 0000000000..5ee34cfcda
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_audit_syncts
@@ -0,0 +1,674 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "audit-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "actor_type",
+              "expr": "$.actor.type"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_type",
+              "expr": "$.context.cdata[*].type"
+            },
+            {
+              "type": "path",
+              "name": "context_cdata_id",
+              "expr": "$.context.cdata[*].id"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_version",
+              "expr": "$.object.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l2",
+              "expr": "$.object.rollup.l2"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_prevstate",
+              "expr": "$.edata.prevstate"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_object_type",
+              "expr": "$.contentdata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "content_type",
+              "expr": "$.contentdata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "content_language",
+              "expr": "$.contentdata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subjects",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "content_framework",
+              "expr": "$.contentdata.framework"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_status",
+              "expr": "$.contentdata.status"
+            },
+            {
+              "type": "path",
+              "name": "content_last_submitted_on",
+              "expr": "$.contentdata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_published_on",
+              "expr": "$.contentdata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "content_last_updated_on",
+              "expr": "$.contentdata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "content_created_by",
+              "expr": "$.contentdata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "content_created_for",
+              "expr": "$.contentdata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_object_type",
+              "expr": "$.collectiondata.objecttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_media_type",
+              "expr": "$.collectiondata.mediatype"
+            },
+            {
+              "type": "path",
+              "name": "collection_language",
+              "expr": "$.collectiondata.language[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subjects",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_mimetype",
+              "expr": "$.collectiondata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "collection_framework",
+              "expr": "$.collectiondata.framework"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_status",
+              "expr": "$.collectiondata.status"
+            },
+            {
+              "type": "path",
+              "name": "collection_version",
+              "expr": "$.collectiondata.pkgversion"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_submitted_on",
+              "expr": "$.collectiondata.lastsubmittedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_published_on",
+              "expr": "$.collectiondata.lastpublishedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_last_updated_on",
+              "expr": "$.collectiondata.lastupdatedon"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_by",
+              "expr": "$.collectiondata.createdby"
+            },
+            {
+              "type": "path",
+              "name": "collection_created_for",
+              "expr": "$.collectiondata.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "device_first_access",
+              "expr": "$.devicedata.firstaccess"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_created_for",
+              "expr": "$.l2data.createdfor"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_framework",
+              "expr": "$.l2data.framework"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_name",
+              "expr": "$.l2data.name"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_channel",
+              "expr": "$.l2data.channel"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_mimetype",
+              "expr": "$.l2data.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_medium",
+              "expr": "$.l2data.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_board",
+              "expr": "$.l2data.board"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_contenttype",
+              "expr": "$.l2data.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_gradelevel",
+              "expr": "$.l2data.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "object_l2_subjects",
+              "expr": "$.l2data.subject[*]"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "actor_id"
+            },
+            {
+              "type": "string",
+              "name": "actor_type"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "name": "context_cdata_type"
+            },
+            {
+              "name": "context_cdata_id"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_version"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l2"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "string",
+              "name": "edata_prevstate"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_object_type"
+            },
+            {
+              "type": "string",
+              "name": "content_type"
+            },
+            {
+              "name": "content_language"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subjects"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "content_framework"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "type": "string",
+              "name": "content_status"
+            },
+            {
+              "type": "long",
+              "name": "content_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "content_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "content_created_by"
+            },
+            {
+              "name": "content_created_for"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_object_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "name": "collection_language"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subjects"
+            },
+            {
+              "type": "string",
+              "name": "collection_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "collection_framework"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "type": "string",
+              "name": "collection_status"
+            },
+            {
+              "type": "double",
+              "name": "collection_version"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_submitted_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_published_on"
+            },
+            {
+              "type": "long",
+              "name": "collection_last_updated_on"
+            },
+            {
+              "type": "string",
+              "name": "collection_created_by"
+            },
+            {
+              "name": "collection_created_for"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "long",
+              "name": "device_first_access"
+            },
+            {
+              "type": "string",
+              "name": "user_declared_state"
+            },
+            {
+              "type": "string",
+              "name": "device_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "name": "object_l2_created_for"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_framework"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_name"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_channel"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_mimetype"
+            },
+            {
+              "name": "object_l2_medium"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_board"
+            },
+            {
+              "type": "string",
+              "name": "object_l2_contenttype"
+            },
+            {
+              "name": "object_l2_gradelevel"
+            },
+            {
+              "name": "object_l2_subjects"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "transformSpec": {
+      "filter": {
+        "type": "selector",
+        "dimension": "eid",
+        "value": "AUDIT"
+      }
+    },
+    "metricsSpec": [
+      {
+        "name": "total_count",
+        "type": "count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_duration",
+        "fieldName": "edata_duration"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "rollup": true
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{rollup_telemetry_audit_syncts_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT14400S",
+    "useEarliestOffset": false,
+    "completionTimeout": "PT1800S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+     "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts
new file mode 100644
index 0000000000..0a515b106c
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_hourly_syncts
@@ -0,0 +1,420 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "telemetry-hourly-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subject",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subject",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_id",
+              "expr": "$.edata.item.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_title",
+              "expr": "$.edata.item.title"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_mode",
+              "expr": "$.edata.mode"
+            },
+            {
+              "type": "path",
+              "name": "edata_size",
+              "expr": "$.edata.size"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "edata_score",
+              "expr": "$.edata.score"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_maxscore",
+              "expr": "$.edata.item.maxscore"
+            },
+            {
+              "type": "path",
+              "name": "edata_rating",
+              "expr": "$.edata.rating"
+            },
+            {
+              "type": "path",
+              "name": "edata_timespent",
+              "expr": "$.edata.timespent"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageviews",
+              "expr": "$.edata.pageviews"
+            },
+            {
+              "type": "path",
+              "name": "edata_interactions",
+              "expr": "$.edata.interactions"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subject"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subject"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_mode"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_title"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "long",
+              "name": "edata_size"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "total_count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_duration",
+        "fieldName": "edata_duration"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_rating",
+        "fieldName": "edata_rating"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_score",
+        "fieldName": "edata_score"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_max_score",
+        "fieldName": "edata_item_maxscore"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_timespent",
+        "fieldName": "edata_timespent"
+      },
+      {
+        "type": "longSum",
+        "name": "total_edata_pageviews",
+        "fieldName": "edata_pageviews"
+      },
+      {
+        "type": "longSum",
+        "name": "total_edata_interactions",
+        "fieldName": "edata_interactions"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "HOUR",
+      "rollup": true
+    },
+    "transformSpec": {
+      "transforms": [
+        {
+          "type": "expression",
+          "name": "edata_size",
+          "expression": "if((edata_size>0),1,0)"
+        }
+      ]
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "maxRowsPerSegment": 5000000
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "replicas": 1,
+    "taskCount": {{rollup_telemetry_hourly_syncts_taskcount}},
+    "taskDuration": "PT3600S",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "completionTimeout": "PT1800S"
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts b/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts
new file mode 100644
index 0000000000..3fc476ff61
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_telemetry_syncts
@@ -0,0 +1,414 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "telemetry-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "path",
+              "name": "context_channel",
+              "expr": "$.context.channel"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_id",
+              "expr": "$.context.pdata.id"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_pid",
+              "expr": "$.context.pdata.pid"
+            },
+            {
+              "type": "path",
+              "name": "context_pdata_ver",
+              "expr": "$.context.pdata.ver"
+            },
+            {
+              "type": "path",
+              "name": "object_id",
+              "expr": "$.object.id"
+            },
+            {
+              "type": "path",
+              "name": "object_type",
+              "expr": "$.object.type"
+            },
+            {
+              "type": "path",
+              "name": "object_rollup_l1",
+              "expr": "$.object.rollup.l1"
+            },
+            {
+              "type": "path",
+              "name": "content_name",
+              "expr": "$.contentdata.name"
+            },
+            {
+              "type": "path",
+              "name": "content_board",
+              "expr": "$.contentdata.board"
+            },
+            {
+              "type": "path",
+              "name": "content_medium",
+              "expr": "$.contentdata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_gradelevel",
+              "expr": "$.contentdata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_subject",
+              "expr": "$.contentdata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "content_mimetype",
+              "expr": "$.contentdata.mimetype"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_state",
+              "expr": "$.derivedlocationdata.state"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_district",
+              "expr": "$.derivedlocationdata.district"
+            },
+            {
+              "type": "path",
+              "name": "derived_loc_from",
+              "expr": "$.derivedlocationdata.from"
+            },
+            {
+              "type": "path",
+              "name": "collection_name",
+              "expr": "$.collectiondata.name"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "collection_board",
+              "expr": "$.collectiondata.board"
+            },
+            {
+              "type": "path",
+              "name": "collection_medium",
+              "expr": "$.collectiondata.medium[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_subject",
+              "expr": "$.collectiondata.subject[*]"
+            },
+            {
+              "type": "path",
+              "name": "collection_gradelevel",
+              "expr": "$.collectiondata.gradelevel[*]"
+            },
+            {
+              "type": "path",
+              "name": "dialcode_channel",
+              "expr": "$.dialcodedata.channel"
+            },
+            {
+              "type": "path",
+              "name": "user_type",
+              "expr": "$.userdata.usertype"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "user_login_type",
+              "expr": "$.userdata.userlogintype"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_id",
+              "expr": "$.edata.item.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_title",
+              "expr": "$.edata.item.title"
+            },
+            {
+              "type": "path",
+              "name": "edata_state",
+              "expr": "$.edata.state"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "edata_size",
+              "expr": "$.edata.size"
+            },
+            {
+              "type": "path",
+              "name": "edata_duration",
+              "expr": "$.edata.duration"
+            },
+            {
+              "type": "path",
+              "name": "edata_score",
+              "expr": "$.edata.score"
+            },
+            {
+              "type": "path",
+              "name": "edata_item_maxscore",
+              "expr": "$.edata.item.maxscore"
+            },
+            {
+              "type": "path",
+              "name": "edata_rating",
+              "expr": "$.edata.rating"
+            },
+            {
+              "type": "path",
+              "name": "edata_timespent",
+              "expr": "$.edata.timespent"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageviews",
+              "expr": "$.edata.pageviews"
+            },
+            {
+              "type": "path",
+              "name": "edata_interactions",
+              "expr": "$.edata.interactions"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "context_channel"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_id"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_pid"
+            },
+            {
+              "type": "string",
+              "name": "context_pdata_ver"
+            },
+            {
+              "type": "string",
+              "name": "object_id"
+            },
+            {
+              "type": "string",
+              "name": "object_type"
+            },
+            {
+              "type": "string",
+              "name": "object_rollup_l1"
+            },
+            {
+              "type": "string",
+              "name": "content_name"
+            },
+            {
+              "type": "string",
+              "name": "content_board"
+            },
+            {
+              "name": "content_medium"
+            },
+            {
+              "name": "content_gradelevel"
+            },
+            {
+              "name": "content_subject"
+            },
+            {
+              "type": "string",
+              "name": "content_mimetype"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_state"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_district"
+            },
+            {
+              "type": "string",
+              "name": "derived_loc_from"
+            },
+            {
+              "type": "string",
+              "name": "collection_name"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_board"
+            },
+            {
+              "name": "collection_medium"
+            },
+            {
+              "name": "collection_gradelevel"
+            },
+            {
+              "name": "collection_subject"
+            },
+            {
+              "type": "string",
+              "name": "dialcode_channel"
+            },
+            {
+              "type": "string",
+              "name": "user_type"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "user_login_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_item_title"
+            },
+            {
+              "type": "string",
+              "name": "edata_state"
+            },
+            {
+              "type": "long",
+              "name": "edata_size"
+            }
+          ],
+          "dimensionsExclusions": []
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "metricsSpec": [
+      {
+        "type": "count",
+        "name": "total_count"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_duration",
+        "fieldName": "edata_duration"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_rating",
+        "fieldName": "edata_rating"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_score",
+        "fieldName": "edata_score"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_max_score",
+        "fieldName": "edata_item_maxscore"
+      },
+      {
+        "type": "doubleSum",
+        "name": "total_edata_timespent",
+        "fieldName": "edata_timespent"
+      },
+      {
+        "type": "longSum",
+        "name": "total_edata_pageviews",
+        "fieldName": "edata_pageviews"
+      },
+      {
+        "type": "longSum",
+        "name": "total_edata_interactions",
+        "fieldName": "edata_interactions"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "DAY",
+      "rollup": true
+    },
+    "transformSpec": {
+      "filter": null,
+      "transforms": [
+        {
+          "type": "expression",
+          "name": "edata_size",
+          "expression": "if((edata_size>0),1,0)"
+        }
+      ]
+    }
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "maxRowsPerSegment": 10000000
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "replicas": 1,
+    "taskCount": {{rollup_telemetry_syncts_taskcount}},
+    "taskDuration": "PT14400S",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "useEarliestOffset": false,
+    "completionTimeout": "PT1800S"
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts b/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts
new file mode 100644
index 0000000000..5878dab401
--- /dev/null
+++ b/ansible/roles/druid-ingestion/templates/rollup_tpd_hourly_syncts
@@ -0,0 +1,165 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "tpd-hourly-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "edata_id",
+              "expr": "$.edata.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_mode",
+              "expr": "$.edata.mode"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageid",
+              "expr": "$.edata.pageid"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "context_did",
+              "expr": "$.context.did"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_mode"
+            },
+            {
+              "type": "string",
+              "name": "edata_pageid"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type":"string",
+              "name": "first_time_user"
+            }
+          ]
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "transformSpec": {
+      "transforms": [
+        {
+          "type": "expression",
+          "name": "first_time_user  ",
+          "expression": "if(like(\"context_cdata_type\",'FirstTimeUser') && like(\"context_cdata_id\",'true') , 'true','false')"
+        },
+        {
+          "type": "expression",
+          "name": "is_playerstart_event",
+          "expression": "if((like(\"context_pdata_pid\",'%contentplayer%'), if ( \"eid\" == 'START', 'true','false') ,'true')"
+        }
+      ],
+      "filter": {
+        "type": "selector",
+        "dimension": "is_playerstart_event",
+        "value": "true"
+      }
+    },
+    "metricsSpec": [
+      {
+        "name": "total_count",
+        "type": "count"
+      },
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_devices",
+        "fieldName": "context_did"
+      },
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_users",
+        "fieldName": "actor_id"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "hour",
+      "rollup": true
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": {{rollup_tpd_hourly_taskcount}},
+    "replicas": 1,
+    "taskDuration": "PT7200S",
+    "completionTimeout" : "PT3600S"
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+     "maxRowsPerSegment": 5000000
+  }
+}
diff --git a/ansible/roles/druid-ingestion/templates/summary_index_kafka b/ansible/roles/druid-ingestion/templates/summary_index_kafka
deleted file mode 100644
index 5506678c29..0000000000
--- a/ansible/roles/druid-ingestion/templates/summary_index_kafka
+++ /dev/null
@@ -1,1124 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "summary-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "ver"
-            },
-            {
-              "type": "root",
-              "name": "ets"
-            },
-            {
-              "type": "root",
-              "name": "uid"
-            },
-            {
-              "type": "root",
-              "name": "syncts"
-            },
-            {
-              "type": "path",
-              "name": "context_date_range_from",
-              "expr": "$.context.date_range.from"
-            },
-            {
-              "type": "path",
-              "name": "context_date_range_to",
-              "expr": "$.context.date_range.to"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "dimension_channel",
-              "expr": "$.dimensions.channel"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_did",
-              "expr": "$.dimensions.did"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_pdata_id",
-              "expr": "$.dimensions.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_pdata_pid",
-              "expr": "$.dimensions.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_pdata_ver",
-              "expr": "$.dimensions.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_sid",
-              "expr": "$.dimensions.sid"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_type",
-              "expr": "$.dimensions.type"
-            },
-            {
-              "type": "path",
-              "name": "dimensions_mode",
-              "expr": "$.dimensions.mode"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_time_spent",
-              "expr": "$.edata.eks.time_spent"
-            },
-            {
-              "type": "path",
-              "name": "edata_time_difference",
-              "expr": "$.edata.eks.time_diff"
-            },
-            {
-              "type": "path",
-              "name": "edata_interaction_count",
-              "expr": "$.edata.eks.interact_events_count"
-            },
-            {
-              "type": "path",
-              "name": "edata_env_summary_env",
-              "expr": "$.edata.eks.env_summary[*].env"
-            },
-            {
-              "type": "path",
-              "name": "edata_env_summary_count",
-              "expr": "$.edata.eks.env_summary[*].count"
-            },
-            {
-              "type": "path",
-              "name": "edata_env_summary_time_spent",
-              "expr": "$.edata.eks.env_summary[*].time_spent"
-            },
-            {
-              "type": "path",
-              "name": "edata_page_summary_id",
-              "expr": "$.edata.eks.page_summary[*].id"
-            },
-            {
-              "type": "path",
-              "name": "edata_page_summary_type",
-              "expr": "$.edata.eks.page_summary[*].type"
-            },
-            {
-              "type": "path",
-              "name": "edata_page_summary_env",
-              "expr": "$.edata.eks.page_summary[*].env"
-            },
-            {
-              "type": "path",
-              "name": "edata_page_summary_visit_count",
-              "expr": "$.edata.eks.page_summary[*].visit_count"
-            },
-            {
-              "type": "path",
-              "name": "edata_page_summary_time_spent",
-              "expr": "$.edata.eks.page_summary[*].time_spent"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_item_id",
-              "expr": "$.edata.eks.item_responses[*].itemId"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_time_spent",
-              "expr": "$.edata.eks.item_responses[*].timeSpent"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_pass",
-              "expr": "$.edata.eks.item_responses[*].pass"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_score",
-              "expr": "$.edata.eks.item_responses[*].score"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_max_score",
-              "expr": "$.edata.eks.item_responses[*].maxScore"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_responses_timestamp",
-              "expr": "$.edata.eks.item_responses[*].time_stamp"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state",
-              "expr": "$.devicedata.state"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_code",
-              "expr": "$.devicedata.statecode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_city",
-              "expr": "$.devicedata.city"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country_code",
-              "expr": "$.devicedata.countrycode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country",
-              "expr": "$.devicedata.country"
-            },
-            {
-              "type": "path",
-              "name": "device_os",
-              "expr": "$.devicedata.devicespec.os"
-            },
-            {
-              "type": "path",
-              "name": "device_make",
-              "expr": "$.devicedata.devicespec.make"
-            },
-            {
-              "type": "path",
-              "name": "device_id",
-              "expr": "$.devicedata.devicespec.id"
-            },
-            {
-              "type": "path",
-              "name": "device_mem",
-              "expr": "$.devicedata.devicespec.mem"
-            },
-            {
-              "type": "path",
-              "name": "device_idisk",
-              "expr": "$.devicedata.devicespec.idisk"
-            },
-            {
-              "type": "path",
-              "name": "device_edisk",
-              "expr": "$.devicedata.devicespec.edisk"
-            },
-            {
-              "type": "path",
-              "name": "device_scrn",
-              "expr": "$.devicedata.devicespec.scrn"
-            },
-            {
-              "type": "path",
-              "name": "device_camera",
-              "expr": "$.devicedata.devicespec.camera"
-            },
-            {
-              "type": "path",
-              "name": "device_cpu",
-              "expr": "$.devicedata.devicespec.cpu"
-            },
-            {
-              "type": "path",
-              "name": "device_sims",
-              "expr": "$.devicedata.devicespec.sims"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_agent",
-              "expr": "$.devicedata.uaspec.agent"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_ver",
-              "expr": "$.devicedata.uaspec.ver"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_system",
-              "expr": "$.devicedata.uaspec.system"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_platform",
-              "expr": "$.devicedata.uaspec.platform"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_raw",
-              "expr": "$.devicedata.uaspec.raw"
-            },
-            {
-              "type": "path",
-              "name": "device_first_access",
-              "expr": "$.devicedata.firstaccess"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_code",
-              "expr": "$.devicedata.statecustomcode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_name",
-              "expr": "$.devicedata.statecustomname"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_district",
-              "expr": "$.devicedata.districtcustom"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_state",
-              "expr": "$.devicedata.userdeclared.state"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_district",
-              "expr": "$.devicedata.userdeclared.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_state",
-              "expr": "$.derivedlocationdata.state"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_district",
-              "expr": "$.derivedlocationdata.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_from",
-              "expr": "$.derivedlocationdata.from"
-            },
-            {
-              "type": "path",
-              "name": "content_name",
-              "expr": "$.contentdata.name"
-            },
-            {
-              "type": "path",
-              "name": "content_object_type",
-              "expr": "$.contentdata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "content_type",
-              "expr": "$.contentdata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "content_media_type",
-              "expr": "$.contentdata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "content_language",
-              "expr": "$.contentdata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_medium",
-              "expr": "$.contentdata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_gradelevel",
-              "expr": "$.contentdata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_subjects",
-              "expr": "$.contentdata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_mimetype",
-              "expr": "$.contentdata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "content_framework",
-              "expr": "$.contentdata.framework"
-            },
-            {
-              "type": "path",
-              "name": "content_board",
-              "expr": "$.contentdata.board"
-            },
-            {
-              "type": "path",
-              "name": "content_status",
-              "expr": "$.contentdata.status"
-            },
-            {
-              "type": "path",
-              "name": "content_created_by",
-              "expr": "$.contentdata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "content_created_for",
-              "expr": "$.contentdata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "content_version",
-              "expr": "$.contentdata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "content_last_submitted_on",
-              "expr": "$.contentdata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_published_on",
-              "expr": "$.contentdata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_updated_on",
-              "expr": "$.contentdata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_name",
-              "expr": "$.collectiondata.name"
-            },
-            {
-              "type": "path",
-              "name": "collection_object_type",
-              "expr": "$.collectiondata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_type",
-              "expr": "$.collectiondata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_media_type",
-              "expr": "$.collectiondata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "collection_language",
-              "expr": "$.collectiondata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_medium",
-              "expr": "$.collectiondata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_gradelevel",
-              "expr": "$.collectiondata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_subjects",
-              "expr": "$.collectiondata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_mimetype",
-              "expr": "$.collectiondata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "collection_framework",
-              "expr": "$.collectiondata.framework"
-            },
-            {
-              "type": "path",
-              "name": "collection_board",
-              "expr": "$.collectiondata.board"
-            },
-            {
-              "type": "path",
-              "name": "collection_status",
-              "expr": "$.collectiondata.status"
-            },
-            {
-              "type": "path",
-              "name": "collection_version",
-              "expr": "$.collectiondata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_submitted_on",
-              "expr": "$.collectiondata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_published_on",
-              "expr": "$.collectiondata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_updated_on",
-              "expr": "$.collectiondata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_by",
-              "expr": "$.collectiondata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_for",
-              "expr": "$.collectiondata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "user_grade_list",
-              "expr": "$.userdata.gradelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_language_list",
-              "expr": "$.userdata.languagelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_subject_list",
-              "expr": "$.userdata.subjectlist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_type",
-              "expr": "$.userdata.usertype"
-            },
-            {
-              "type": "path",
-              "name": "user_roles",
-              "expr": "$.userdata.roles[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_state",
-              "expr": "$.userdata.state"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_district",
-              "expr": "$.userdata.district"
-            },
-            {
-              "type": "path",
-              "name": "user_signin_type",
-              "expr": "$.userdata.usersignintype"
-            },
-            {
-              "type": "path",
-              "name": "user_login_type",
-              "expr": "$.userdata.userlogintype"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_iso_state_code",
-              "expr": "$.devicedata.iso3166statecode"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "string",
-              "name": "ver"
-            },
-            {
-              "type": "long",
-              "name": "syncts"
-            },
-            {
-              "type": "string",
-              "name": "uid"
-            },
-            {
-              "type": "long",
-              "name": "context_date_range_from"
-            },
-            {
-              "name": "context_cdata_type"
-            },
-            {
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "dimension_channel"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_did"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_sid"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_type"
-            },
-            {
-              "type": "string",
-              "name": "dimensions_mode"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "name": "tags"
-            },
-            {
-              "type": "double",
-              "name": "edata_time_spent"
-            },
-            {
-              "type": "double",
-              "name": "edata_time_difference"
-            },
-            {
-              "type": "long",
-              "name": "edata_interaction_count"
-            },
-            {
-              "type": "string",
-              "name": "edata_env_summary_env"
-            },
-            {
-              "type": "string",
-              "name": "edata_env_summary_count"
-            },
-            {
-              "type": "string",
-              "name": "edata_env_summary_time_spent"
-            },
-            {
-              "type": "string",
-              "name": "edata_page_summary_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_page_summary_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_page_summary_env"
-            },
-            {
-              "type": "string",
-              "name": "edata_page_summary_visit_count"
-            },
-            {
-              "type": "string",
-              "name": "edata_page_summary_time_spent"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_item_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_time_spent"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_pass"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_score"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_max_score"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_responses_timestamp"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_city"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country"
-            },
-            {
-              "type": "string",
-              "name": "device_os"
-            },
-            {
-              "type": "string",
-              "name": "device_make"
-            },
-            {
-              "type": "string",
-              "name": "device_id"
-            },
-            {
-              "type": "long",
-              "name": "device_mem"
-            },
-            {
-              "type": "string",
-              "name": "device_idisk"
-            },
-            {
-              "type": "string",
-              "name": "device_edisk"
-            },
-            {
-              "type": "string",
-              "name": "device_scrn"
-            },
-            {
-              "type": "string",
-              "name": "device_camera"
-            },
-            {
-              "type": "string",
-              "name": "device_cpu"
-            },
-            {
-              "type": "long",
-              "name": "device_sims"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_agent"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_ver"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_system"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_platform"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_raw"
-            },
-            {
-              "type": "long",
-              "name": "device_first_access"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_name"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_state"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_from"
-            },
-            {
-              "type": "string",
-              "name": "content_name"
-            },
-            {
-              "type": "string",
-              "name": "content_object_type"
-            },
-            {
-              "type": "string",
-              "name": "content_type"
-            },
-            {
-              "type": "string",
-              "name": "content_media_type"
-            },
-            {
-              "name": "content_language"
-            },
-            {
-              "name": "content_medium"
-            },
-            {
-              "name": "content_gradelevel"
-            },
-            {
-              "name": "content_subjects"
-            },
-            {
-              "type": "string",
-              "name": "content_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "content_framework"
-            },
-            {
-              "type": "string",
-              "name": "content_board"
-            },
-            {
-              "type": "string",
-              "name": "content_status"
-            },
-            {
-              "type": "double",
-              "name": "content_version"
-            },
-            {
-              "type": "long",
-              "name": "content_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "content_created_by"
-            },
-            {
-              "name": "content_created_for"
-            },
-            {
-              "type": "string",
-              "name": "collection_name"
-            },
-            {
-              "type": "string",
-              "name": "collection_object_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_media_type"
-            },
-            {
-              "name": "collection_language"
-            },
-            {
-              "name": "collection_medium"
-            },
-            {
-              "name": "collection_gradelevel"
-            },
-            {
-              "name": "collection_subjects"
-            },
-            {
-              "type": "string",
-              "name": "collection_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "collection_framework"
-            },
-            {
-              "type": "string",
-              "name": "collection_board"
-            },
-            {
-              "type": "string",
-              "name": "collection_status"
-            },
-            {
-              "type": "double",
-              "name": "collection_version"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "collection_created_by"
-            },
-            {
-              "name": "collection_created_for"
-            },
-            {
-              "name": "user_grade_list"
-            },
-            {
-              "name": "user_language_list"
-            },
-            {
-              "name": "user_subject_list"
-            },
-            {
-              "type": "string",
-              "name": "user_type"
-            },
-            {
-              "name": "user_roles"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_signin_type"
-            },
-            {
-              "type": "string",
-              "name": "user_login_type"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_iso_state_code"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "syncts",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [
-      {
-        "type": "longSum",
-        "name": "total_interactions",
-        "fieldName": "edata_interaction_count"
-      },
-      {
-        "type": "doubleSum",
-        "name": "total_time_spent",
-        "fieldName": "edata_time_spent"
-      }
-    ],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": true
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.summary",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka b/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka
deleted file mode 100644
index c9a86819de..0000000000
--- a/ansible/roles/druid-ingestion/templates/telemetry_feedback_index_kafka
+++ /dev/null
@@ -1,777 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "telemetry-feedback-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "ets"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_rating",
-              "expr": "$.edata.rating"
-            },
-            {
-              "type": "path",
-              "name": "edata_comments",
-              "expr": "$.edata.comments"
-            },
-            {
-              "type": "path",
-              "name": "edata_commentid",
-              "expr": "$.edata.commentid"
-            },
-            {
-              "type": "path",
-              "name": "edata_commenttxt",
-              "expr": "$.edata.commenttxt"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state",
-              "expr": "$.devicedata.state"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_code",
-              "expr": "$.devicedata.statecode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_city",
-              "expr": "$.devicedata.city"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country_code",
-              "expr": "$.devicedata.countrycode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country",
-              "expr": "$.devicedata.country"
-            },
-            {
-              "type": "path",
-              "name": "device_os",
-              "expr": "$.devicedata.devicespec.os"
-            },
-            {
-              "type": "path",
-              "name": "device_make",
-              "expr": "$.devicedata.devicespec.make"
-            },
-            {
-              "type": "path",
-              "name": "device_id",
-              "expr": "$.devicedata.devicespec.id"
-            },
-            {
-              "type": "path",
-              "name": "device_mem",
-              "expr": "$.devicedata.devicespec.mem"
-            },
-            {
-              "type": "path",
-              "name": "device_idisk",
-              "expr": "$.devicedata.devicespec.idisk"
-            },
-            {
-              "type": "path",
-              "name": "device_edisk",
-              "expr": "$.devicedata.devicespec.edisk"
-            },
-            {
-              "type": "path",
-              "name": "device_scrn",
-              "expr": "$.devicedata.devicespec.scrn"
-            },
-            {
-              "type": "path",
-              "name": "device_camera",
-              "expr": "$.devicedata.devicespec.camera"
-            },
-            {
-              "type": "path",
-              "name": "device_cpu",
-              "expr": "$.devicedata.devicespec.cpu"
-            },
-            {
-              "type": "path",
-              "name": "device_sims",
-              "expr": "$.devicedata.devicespec.sims"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_agent",
-              "expr": "$.devicedata.uaspec.agent"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_ver",
-              "expr": "$.devicedata.uaspec.ver"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_system",
-              "expr": "$.devicedata.uaspec.system"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_platform",
-              "expr": "$.devicedata.uaspec.platform"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_raw",
-              "expr": "$.devicedata.uaspec.raw"
-            },
-            {
-              "type": "path",
-              "name": "device_first_access",
-              "expr": "$.devicedata.firstaccess"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_code",
-              "expr": "$.devicedata.statecustomcode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_name",
-              "expr": "$.devicedata.statecustomname"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_district",
-              "expr": "$.devicedata.districtcustom"
-            },
-            {
-              "type": "path",
-              "name": "content_name",
-              "expr": "$.contentdata.name"
-            },
-            {
-              "type": "path",
-              "name": "content_object_type",
-              "expr": "$.contentdata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "content_type",
-              "expr": "$.contentdata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "content_media_type",
-              "expr": "$.contentdata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "content_language",
-              "expr": "$.contentdata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_medium",
-              "expr": "$.contentdata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_gradelevel",
-              "expr": "$.contentdata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_subjects",
-              "expr": "$.contentdata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_mimetype",
-              "expr": "$.contentdata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "content_framework",
-              "expr": "$.contentdata.framework"
-            },
-            {
-              "type": "path",
-              "name": "content_board",
-              "expr": "$.contentdata.board"
-            },
-            {
-              "type": "path",
-              "name": "content_status",
-              "expr": "$.contentdata.status"
-            },
-            {
-              "type": "path",
-              "name": "content_version",
-              "expr": "$.contentdata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "content_last_submitted_on",
-              "expr": "$.contentdata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_published_on",
-              "expr": "$.contentdata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_updated_on",
-              "expr": "$.contentdata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "user_grade_list",
-              "expr": "$.userdata.gradelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_language_list",
-              "expr": "$.userdata.languagelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_subject_list",
-              "expr": "$.userdata.subjectlist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_type",
-              "expr": "$.userdata.usertype"
-            },
-            {
-              "type": "path",
-              "name": "user_roles",
-              "expr": "$.userdata.roles[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_state",
-              "expr": "$.userdata.state"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_district",
-              "expr": "$.userdata.district"
-            },
-            {
-              "type": "path",
-              "name": "user_signin_type",
-              "expr": "$.userdata.usersignintype"
-            },
-            {
-              "type": "path",
-              "name": "user_login_type",
-              "expr": "$.userdata.userlogintype"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_iso_state_code",
-              "expr": "$.devicedata.iso3166statecode"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "long",
-              "name": "ets"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "name": "context_cdata_type"
-            },
-            {
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "name": "tags"
-            },
-            {
-              "type": "long",
-              "name": "edata_rating"
-            },
-            {
-              "type": "string",
-              "name": "edata_comments"
-            },
-            {
-              "type": "string",
-              "name": "edata_commentid"
-            },
-            {
-              "type": "string",
-              "name": "edata_commenttxt"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_city"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country"
-            },
-            {
-              "type": "string",
-              "name": "device_os"
-            },
-            {
-              "type": "string",
-              "name": "device_make"
-            },
-            {
-              "type": "string",
-              "name": "device_id"
-            },
-            {
-              "type": "long",
-              "name": "device_mem"
-            },
-            {
-              "type": "string",
-              "name": "device_idisk"
-            },
-            {
-              "type": "string",
-              "name": "device_edisk"
-            },
-            {
-              "type": "string",
-              "name": "device_scrn"
-            },
-            {
-              "type": "string",
-              "name": "device_camera"
-            },
-            {
-              "type": "string",
-              "name": "device_cpu"
-            },
-            {
-              "type": "long",
-              "name": "device_sims"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_agent"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_ver"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_system"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_platform"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_raw"
-            },
-            {
-              "type": "long",
-              "name": "device_first_access"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_name"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "content_name"
-            },
-            {
-              "type": "string",
-              "name": "content_object_type"
-            },
-            {
-              "type": "string",
-              "name": "content_type"
-            },
-            {
-              "type": "string",
-              "name": "content_media_type"
-            },
-            {
-              "name": "content_language"
-            },
-            {
-              "name": "content_medium"
-            },
-            {
-              "name": "content_gradelevel"
-            },
-            {
-              "name": "content_subjects"
-            },
-            {
-              "type": "string",
-              "name": "content_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "content_framework"
-            },
-            {
-              "type": "string",
-              "name": "content_board"
-            },
-            {
-              "type": "string",
-              "name": "content_status"
-            },
-            {
-              "type": "double",
-              "name": "content_version"
-            },
-            {
-              "type": "long",
-              "name": "content_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_updated_on"
-            },
-            {
-              "name": "user_grade_list"
-            },
-            {
-              "name": "user_language_list"
-            },
-            {
-              "name": "user_subject_list"
-            },
-            {
-              "type": "string",
-              "name": "user_type"
-            },
-            {
-              "name": "user_roles"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_signin_type"
-            },
-            {
-              "type": "string",
-              "name": "user_login_type"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_iso_state_code"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "syncts",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [
-      {
-        "type": "count",
-        "name": "count"
-      },
-      {
-        "type": "longSum",
-        "name": "total_rating",
-        "fieldName": "edata_rating"
-      }
-    ],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": true
-    },
-    "transformSpec": {
-      "filter": {
-        "type": "selector",
-        "dimension": "eid",
-        "value": "FEEDBACK"
-      }
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.telemetry",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/telemetry_index_kafka b/ansible/roles/druid-ingestion/templates/telemetry_index_kafka
deleted file mode 100644
index daf253e287..0000000000
--- a/ansible/roles/druid-ingestion/templates/telemetry_index_kafka
+++ /dev/null
@@ -1,1362 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "telemetry-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "syncts"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "content_play_progress",
-              "expr": "$.edata.summary[*].progress"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_type",
-              "expr": "$.edata.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_subtype",
-              "expr": "$.edata.subtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_mode",
-              "expr": "$.edata.mode"
-            },
-            {
-              "type": "path",
-              "name": "edata_pageid",
-              "expr": "$.edata.pageid"
-            },
-            {
-              "type": "path",
-              "name": "edata_uri",
-              "expr": "$.edata.uri"
-            },
-            {
-              "type": "path",
-              "name": "edata_id",
-              "expr": "$.edata.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_duration",
-              "expr": "$.edata.duration"
-            },
-            {
-              "type": "path",
-              "name": "edata_index",
-              "expr": "$.edata.index"
-            },
-            {
-              "type": "path",
-              "name": "edata_pass",
-              "expr": "$.edata.pass"
-            },
-            {
-              "type": "path",
-              "name": "edata_score",
-              "expr": "$.edata.score"
-            },
-            {
-              "type": "path",
-              "name": "edata_resvalues",
-              "expr": "$.edata.resvalues[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_id",
-              "expr": "$.edata.item.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_title",
-              "expr": "$.edata.item.title"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_maxscore",
-              "expr": "$.edata.item.maxscore"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_id",
-              "expr": "$.edata.target.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_type",
-              "expr": "$.edata.target.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_rating",
-              "expr": "$.edata.rating"
-            },
-            {
-              "type": "path",
-              "name": "edata_comments",
-              "expr": "$.edata.comments"
-            },
-            {
-              "type": "path",
-              "name": "edata_commentid",
-              "expr": "$.edata.commentid"
-            },
-            {
-              "type": "path",
-              "name": "edata_commenttxt",
-              "expr": "$.edata.commenttxt"
-            },
-            {
-              "type": "path",
-              "name": "edata_dir",
-              "expr": "$.edata.dir"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_id",
-              "expr": "$.edata.items[*].id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_type",
-              "expr": "$.edata.items[*].type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_id",
-              "expr": "$.edata.items[*].origin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_type",
-              "expr": "$.edata.items[*].origin.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_id",
-              "expr": "$.edata.items[*].to.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_type",
-              "expr": "$.edata.items[*].to.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_id",
-              "expr": "$.edata.plugin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_ver",
-              "expr": "$.edata.plugin.ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_category",
-              "expr": "$.edata.plugin.category"
-            },
-            {
-              "type": "path",
-              "name": "edata_props",
-              "expr": "$.edata.props[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_state",
-              "expr": "$.edata.state"
-            },
-            {
-              "type": "path",
-              "name": "edata_prevstate",
-              "expr": "$.edata.prevstate"
-            },
-            {
-              "type": "path",
-              "name": "edata_size",
-              "expr": "$.edata.size"
-            },
-            {
-              "type": "path",
-              "name": "edata_filters_dialcodes",
-              "expr": "$.edata.filters.dialcodes"
-            },
-            {
-              "type": "path",
-              "name": "edata_topn_identifier",
-              "expr": "$.edata.topn[*].identifier"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objid",
-              "expr": "$.edata.visits[*].objid"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objtype",
-              "expr": "$.edata.visits[*].objtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objver",
-              "expr": "$.edata.visits[*].objver"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_index",
-              "expr": "$.edata.visits[*].index"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state",
-              "expr": "$.devicedata.state"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_code",
-              "expr": "$.devicedata.statecode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_city",
-              "expr": "$.devicedata.city"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country_code",
-              "expr": "$.devicedata.countrycode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country",
-              "expr": "$.devicedata.country"
-            },
-            {
-              "type": "path",
-              "name": "device_os",
-              "expr": "$.devicedata.devicespec.os"
-            },
-            {
-              "type": "path",
-              "name": "device_make",
-              "expr": "$.devicedata.devicespec.make"
-            },
-            {
-              "type": "path",
-              "name": "device_id",
-              "expr": "$.devicedata.devicespec.id"
-            },
-            {
-              "type": "path",
-              "name": "device_mem",
-              "expr": "$.devicedata.devicespec.mem"
-            },
-            {
-              "type": "path",
-              "name": "device_idisk",
-              "expr": "$.devicedata.devicespec.idisk"
-            },
-            {
-              "type": "path",
-              "name": "device_edisk",
-              "expr": "$.devicedata.devicespec.edisk"
-            },
-            {
-              "type": "path",
-              "name": "device_scrn",
-              "expr": "$.devicedata.devicespec.scrn"
-            },
-            {
-              "type": "path",
-              "name": "device_camera",
-              "expr": "$.devicedata.devicespec.camera"
-            },
-            {
-              "type": "path",
-              "name": "device_cpu",
-              "expr": "$.devicedata.devicespec.cpu"
-            },
-            {
-              "type": "path",
-              "name": "device_sims",
-              "expr": "$.devicedata.devicespec.sims"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_agent",
-              "expr": "$.devicedata.uaspec.agent"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_ver",
-              "expr": "$.devicedata.uaspec.ver"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_system",
-              "expr": "$.devicedata.uaspec.system"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_platform",
-              "expr": "$.devicedata.uaspec.platform"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_raw",
-              "expr": "$.devicedata.uaspec.raw"
-            },
-            {
-              "type": "path",
-              "name": "device_first_access",
-              "expr": "$.devicedata.firstaccess"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_code",
-              "expr": "$.devicedata.statecustomcode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_name",
-              "expr": "$.devicedata.statecustomname"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_district",
-              "expr": "$.devicedata.districtcustom"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_state",
-              "expr": "$.devicedata.userdeclared.state"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_district",
-              "expr": "$.devicedata.userdeclared.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_state",
-              "expr": "$.derivedlocationdata.state"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_district",
-              "expr": "$.derivedlocationdata.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_from",
-              "expr": "$.derivedlocationdata.from"
-            },
-            {
-              "type": "path",
-              "name": "content_name",
-              "expr": "$.contentdata.name"
-            },
-            {
-              "type": "path",
-              "name": "content_object_type",
-              "expr": "$.contentdata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "content_type",
-              "expr": "$.contentdata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "content_media_type",
-              "expr": "$.contentdata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "content_language",
-              "expr": "$.contentdata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_medium",
-              "expr": "$.contentdata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_gradelevel",
-              "expr": "$.contentdata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_subjects",
-              "expr": "$.contentdata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_mimetype",
-              "expr": "$.contentdata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "content_framework",
-              "expr": "$.contentdata.framework"
-            },
-            {
-              "type": "path",
-              "name": "content_board",
-              "expr": "$.contentdata.board"
-            },
-            {
-              "type": "path",
-              "name": "content_status",
-              "expr": "$.contentdata.status"
-            },
-            {
-              "type": "path",
-              "name": "content_version",
-              "expr": "$.contentdata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "content_last_submitted_on",
-              "expr": "$.contentdata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_published_on",
-              "expr": "$.contentdata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_updated_on",
-              "expr": "$.contentdata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "content_created_by",
-              "expr": "$.contentdata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "content_created_for",
-              "expr": "$.contentdata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "collection_name",
-              "expr": "$.collectiondata.name"
-            },
-            {
-              "type": "path",
-              "name": "collection_object_type",
-              "expr": "$.collectiondata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_type",
-              "expr": "$.collectiondata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_media_type",
-              "expr": "$.collectiondata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "collection_language",
-              "expr": "$.collectiondata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_medium",
-              "expr": "$.collectiondata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_gradelevel",
-              "expr": "$.collectiondata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_subjects",
-              "expr": "$.collectiondata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_mimetype",
-              "expr": "$.collectiondata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "collection_framework",
-              "expr": "$.collectiondata.framework"
-            },
-            {
-              "type": "path",
-              "name": "collection_board",
-              "expr": "$.collectiondata.board"
-            },
-            {
-              "type": "path",
-              "name": "collection_status",
-              "expr": "$.collectiondata.status"
-            },
-            {
-              "type": "path",
-              "name": "collection_version",
-              "expr": "$.collectiondata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_submitted_on",
-              "expr": "$.collectiondata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_published_on",
-              "expr": "$.collectiondata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_updated_on",
-              "expr": "$.collectiondata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_by",
-              "expr": "$.collectiondata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_for",
-              "expr": "$.collectiondata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "user_grade_list",
-              "expr": "$.userdata.gradelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_language_list",
-              "expr": "$.userdata.languagelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_subject_list",
-              "expr": "$.userdata.subjectlist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_type",
-              "expr": "$.userdata.usertype"
-            },
-            {
-              "type": "path",
-              "name": "user_roles",
-              "expr": "$.userdata.roles[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_state",
-              "expr": "$.userdata.state"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_district",
-              "expr": "$.userdata.district"
-            },
-            {
-              "type": "path",
-              "name": "user_signin_type",
-              "expr": "$.userdata.usersignintype"
-            },
-            {
-              "type": "path",
-              "name": "user_login_type",
-              "expr": "$.userdata.userlogintype"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_channel",
-              "expr": "$.dialcodedata.channel"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_batchcode",
-              "expr": "$.dialcodedata.batchcode"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_publisher",
-              "expr": "$.dialcodedata.publisher"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_generated_on",
-              "expr": "$.dialcodedata.generatedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_published_on",
-              "expr": "$.dialcodedata.publishedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_object_type",
-              "expr": "$.dialcodedata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_iso_state_code",
-              "expr": "$.devicedata.iso3166statecode"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "long",
-              "name": "syncts"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "name": "context_cdata_type"
-            },
-            {
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "name": "tags"
-            },
-            {
-              "type": "string",
-              "name": "edata_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_subtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_mode"
-            },
-            {
-              "type": "string",
-              "name": "edata_pageid"
-            },
-            {
-              "type": "string",
-              "name": "edata_uri"
-            },
-            {
-              "type": "string",
-              "name": "edata_id"
-            },
-            {
-              "type": "double",
-              "name": "edata_duration"
-            },
-            {
-              "type": "long",
-              "name": "edata_index"
-            },
-            {
-              "type": "string",
-              "name": "edata_pass"
-            },
-            {
-              "type": "double",
-              "name": "edata_score"
-            },
-            {
-              "name": "edata_resvalues"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_title"
-            },
-            {
-              "type": "double",
-              "name": "edata_item_maxscore"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_type"
-            },
-            {
-              "type": "long",
-              "name": "edata_rating"
-            },
-            {
-              "type": "string",
-              "name": "edata_comments"
-            },
-            {
-              "type": "string",
-              "name": "edata_commentid"
-            },
-            {
-              "type": "string",
-              "name": "edata_commenttxt"
-            },
-            {
-              "type": "string",
-              "name": "edata_dir"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_category"
-            },
-            {
-              "name": "edata_props"
-            },
-            {
-              "type": "string",
-              "name": "edata_state"
-            },
-            {
-              "type": "string",
-              "name": "edata_prevstate"
-            },
-            {
-              "type": "long",
-              "name": "edata_size"
-            },
-            {
-              "name": "edata_filters_dialcodes"
-            },
-            {
-              "type": "string",
-              "name": "edata_topn_identifier"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objid"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objver"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_index"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_city"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country"
-            },
-            {
-              "type": "string",
-              "name": "device_os"
-            },
-            {
-              "type": "string",
-              "name": "device_make"
-            },
-            {
-              "type": "string",
-              "name": "device_id"
-            },
-            {
-              "type": "long",
-              "name": "device_mem"
-            },
-            {
-              "type": "string",
-              "name": "device_idisk"
-            },
-            {
-              "type": "string",
-              "name": "device_edisk"
-            },
-            {
-              "type": "string",
-              "name": "device_scrn"
-            },
-            {
-              "type": "string",
-              "name": "device_camera"
-            },
-            {
-              "type": "string",
-              "name": "device_cpu"
-            },
-            {
-              "type": "long",
-              "name": "device_sims"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_agent"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_ver"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_system"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_platform"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_raw"
-            },
-            {
-              "type": "long",
-              "name": "device_first_access"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_name"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_state"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_from"
-            },
-            {
-              "type": "string",
-              "name": "content_name"
-            },
-            {
-              "type": "string",
-              "name": "content_object_type"
-            },
-            {
-              "type": "string",
-              "name": "content_type"
-            },
-            {
-              "type": "string",
-              "name": "content_media_type"
-            },
-            {
-              "name": "content_language"
-            },
-            {
-              "name": "content_medium"
-            },
-            {
-              "name": "content_gradelevel"
-            },
-            {
-              "name": "content_subjects"
-            },
-            {
-              "type": "string",
-              "name": "content_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "content_framework"
-            },
-            {
-              "type": "string",
-              "name": "content_board"
-            },
-            {
-              "type": "string",
-              "name": "content_status"
-            },
-            {
-              "type": "double",
-              "name": "content_version"
-            },
-            {
-              "type": "long",
-              "name": "content_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "content_created_by"
-            },
-            {
-              "name": "content_created_for"
-            },
-            {
-              "name" : "content_play_progress"
-            },
-            {
-              "type": "string",
-              "name": "collection_name"
-            },
-            {
-              "type": "string",
-              "name": "collection_object_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_media_type"
-            },
-            {
-              "name": "collection_language"
-            },
-            {
-              "name": "collection_medium"
-            },
-            {
-              "name": "collection_gradelevel"
-            },
-            {
-              "name": "collection_subjects"
-            },
-            {
-              "type": "string",
-              "name": "collection_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "collection_framework"
-            },
-            {
-              "type": "string",
-              "name": "collection_board"
-            },
-            {
-              "type": "string",
-              "name": "collection_status"
-            },
-            {
-              "type": "double",
-              "name": "collection_version"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "collection_created_by"
-            },
-            {
-              "name": "collection_created_for"
-            },
-            {
-              "name": "user_grade_list"
-            },
-            {
-              "name": "user_language_list"
-            },
-            {
-              "name": "user_subject_list"
-            },
-            {
-              "type": "string",
-              "name": "user_type"
-            },
-            {
-              "name": "user_roles"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_signin_type"
-            },
-            {
-              "type": "string",
-              "name": "user_login_type"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_channel"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_batchcode"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_publisher"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_generated_on"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_published_on"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_object_type"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_iso_state_code"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "ets",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.telemetry",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka b/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka
deleted file mode 100644
index 04266a46f0..0000000000
--- a/ansible/roles/druid-ingestion/templates/telemetry_index_syncts_kafka
+++ /dev/null
@@ -1,1355 +0,0 @@
-{
-  "type": "kafka",
-  "dataSchema": {
-    "dataSource": "telemetry-events",
-    "parser": {
-      "type": "string",
-      "parseSpec": {
-        "format": "json",
-        "flattenSpec": {
-          "useFieldDiscovery": false,
-          "fields": [
-            {
-              "type": "root",
-              "name": "eid"
-            },
-            {
-              "type": "root",
-              "name": "mid"
-            },
-            {
-              "type": "root",
-              "name": "ets"
-            },
-            {
-              "type": "root",
-              "name": "@timestamp"
-            },
-            {
-              "type": "path",
-              "name": "actor_id",
-              "expr": "$.actor.id"
-            },
-            {
-              "type": "path",
-              "name": "actor_type",
-              "expr": "$.actor.type"
-            },
-            {
-              "type": "path",
-              "name": "context_channel",
-              "expr": "$.context.channel"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_id",
-              "expr": "$.context.pdata.id"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_pid",
-              "expr": "$.context.pdata.pid"
-            },
-            {
-              "type": "path",
-              "name": "context_pdata_ver",
-              "expr": "$.context.pdata.ver"
-            },
-            {
-              "type": "path",
-              "name": "context_env",
-              "expr": "$.context.env"
-            },
-            {
-              "type": "path",
-              "name": "context_sid",
-              "expr": "$.context.sid"
-            },
-            {
-              "type": "path",
-              "name": "context_did",
-              "expr": "$.context.did"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_type",
-              "expr": "$.context.cdata[*].type"
-            },
-            {
-              "type": "path",
-              "name": "context_cdata_id",
-              "expr": "$.context.cdata[*].id"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l1",
-              "expr": "$.context.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l2",
-              "expr": "$.context.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l3",
-              "expr": "$.context.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "context_rollup_l4",
-              "expr": "$.context.rollup.l4"
-            },
-            {
-              "type": "path",
-              "name": "object_id",
-              "expr": "$.object.id"
-            },
-            {
-              "type": "path",
-              "name": "object_type",
-              "expr": "$.object.type"
-            },
-            {
-              "type": "path",
-              "name": "object_version",
-              "expr": "$.object.ver"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l1",
-              "expr": "$.object.rollup.l1"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l2",
-              "expr": "$.object.rollup.l2"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l3",
-              "expr": "$.object.rollup.l3"
-            },
-            {
-              "type": "path",
-              "name": "object_rollup_l4",
-              "expr": "$.object.rollup.l4"
-            },
-            {
-              "type": "root",
-              "name": "tags"
-            },
-            {
-              "type": "path",
-              "name": "edata_type",
-              "expr": "$.edata.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_subtype",
-              "expr": "$.edata.subtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_mode",
-              "expr": "$.edata.mode"
-            },
-            {
-              "type": "path",
-              "name": "edata_pageid",
-              "expr": "$.edata.pageid"
-            },
-            {
-              "type": "path",
-              "name": "edata_uri",
-              "expr": "$.edata.uri"
-            },
-            {
-              "type": "path",
-              "name": "edata_id",
-              "expr": "$.edata.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_duration",
-              "expr": "$.edata.duration"
-            },
-            {
-              "type": "path",
-              "name": "edata_index",
-              "expr": "$.edata.index"
-            },
-            {
-              "type": "path",
-              "name": "edata_pass",
-              "expr": "$.edata.pass"
-            },
-            {
-              "type": "path",
-              "name": "edata_score",
-              "expr": "$.edata.score"
-            },
-            {
-              "type": "path",
-              "name": "edata_resvalues",
-              "expr": "$.edata.resvalues[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_id",
-              "expr": "$.edata.item.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_title",
-              "expr": "$.edata.item.title"
-            },
-            {
-              "type": "path",
-              "name": "edata_item_maxscore",
-              "expr": "$.edata.item.maxscore"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_id",
-              "expr": "$.edata.target.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_target_type",
-              "expr": "$.edata.target.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_rating",
-              "expr": "$.edata.rating"
-            },
-            {
-              "type": "path",
-              "name": "edata_comments",
-              "expr": "$.edata.comments"
-            },
-            {
-              "type": "path",
-              "name": "edata_commentid",
-              "expr": "$.edata.commentid"
-            },
-            {
-              "type": "path",
-              "name": "edata_commenttxt",
-              "expr": "$.edata.commenttxt"
-            },
-            {
-              "type": "path",
-              "name": "edata_dir",
-              "expr": "$.edata.dir"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_id",
-              "expr": "$.edata.items[*].id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_type",
-              "expr": "$.edata.items[*].type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_id",
-              "expr": "$.edata.items[*].origin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_origin_type",
-              "expr": "$.edata.items[*].origin.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_id",
-              "expr": "$.edata.items[*].to.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_items_to_type",
-              "expr": "$.edata.items[*].to.type"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_id",
-              "expr": "$.edata.plugin.id"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_ver",
-              "expr": "$.edata.plugin.ver"
-            },
-            {
-              "type": "path",
-              "name": "edata_plugin_category",
-              "expr": "$.edata.plugin.category"
-            },
-            {
-              "type": "path",
-              "name": "edata_props",
-              "expr": "$.edata.props[*]"
-            },
-            {
-              "type": "path",
-              "name": "edata_state",
-              "expr": "$.edata.state"
-            },
-            {
-              "type": "path",
-              "name": "edata_prevstate",
-              "expr": "$.edata.prevstate"
-            },
-            {
-              "type": "path",
-              "name": "edata_size",
-              "expr": "$.edata.size"
-            },
-            {
-              "type": "path",
-              "name": "edata_filters_dialcodes",
-              "expr": "$.edata.filters.dialcodes"
-            },
-            {
-              "type": "path",
-              "name": "edata_topn_identifier",
-              "expr": "$.edata.topn[*].identifier"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objid",
-              "expr": "$.edata.visits[*].objid"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objtype",
-              "expr": "$.edata.visits[*].objtype"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_objver",
-              "expr": "$.edata.visits[*].objver"
-            },
-            {
-              "type": "path",
-              "name": "edata_visits_index",
-              "expr": "$.edata.visits[*].index"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state",
-              "expr": "$.devicedata.state"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_code",
-              "expr": "$.devicedata.statecode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_city",
-              "expr": "$.devicedata.city"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country_code",
-              "expr": "$.devicedata.countrycode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_country",
-              "expr": "$.devicedata.country"
-            },
-            {
-              "type": "path",
-              "name": "device_os",
-              "expr": "$.devicedata.devicespec.os"
-            },
-            {
-              "type": "path",
-              "name": "device_make",
-              "expr": "$.devicedata.devicespec.make"
-            },
-            {
-              "type": "path",
-              "name": "device_id",
-              "expr": "$.devicedata.devicespec.id"
-            },
-            {
-              "type": "path",
-              "name": "device_mem",
-              "expr": "$.devicedata.devicespec.mem"
-            },
-            {
-              "type": "path",
-              "name": "device_idisk",
-              "expr": "$.devicedata.devicespec.idisk"
-            },
-            {
-              "type": "path",
-              "name": "device_edisk",
-              "expr": "$.devicedata.devicespec.edisk"
-            },
-            {
-              "type": "path",
-              "name": "device_scrn",
-              "expr": "$.devicedata.devicespec.scrn"
-            },
-            {
-              "type": "path",
-              "name": "device_camera",
-              "expr": "$.devicedata.devicespec.camera"
-            },
-            {
-              "type": "path",
-              "name": "device_cpu",
-              "expr": "$.devicedata.devicespec.cpu"
-            },
-            {
-              "type": "path",
-              "name": "device_sims",
-              "expr": "$.devicedata.devicespec.sims"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_agent",
-              "expr": "$.devicedata.uaspec.agent"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_ver",
-              "expr": "$.devicedata.uaspec.ver"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_system",
-              "expr": "$.devicedata.uaspec.system"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_platform",
-              "expr": "$.devicedata.uaspec.platform"
-            },
-            {
-              "type": "path",
-              "name": "device_uaspec_raw",
-              "expr": "$.devicedata.uaspec.raw"
-            },
-            {
-              "type": "path",
-              "name": "device_first_access",
-              "expr": "$.devicedata.firstaccess"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_code",
-              "expr": "$.devicedata.statecustomcode"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_state_custom_name",
-              "expr": "$.devicedata.statecustomname"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_district",
-              "expr": "$.devicedata.districtcustom"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_state",
-              "expr": "$.devicedata.userdeclared.state"
-            },
-            {
-              "type": "path",
-              "name": "user_declared_district",
-              "expr": "$.devicedata.userdeclared.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_state",
-              "expr": "$.derivedlocationdata.state"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_district",
-              "expr": "$.derivedlocationdata.district"
-            },
-            {
-              "type": "path",
-              "name": "derived_loc_from",
-              "expr": "$.derivedlocationdata.from"
-            },
-            {
-              "type": "path",
-              "name": "content_name",
-              "expr": "$.contentdata.name"
-            },
-            {
-              "type": "path",
-              "name": "content_object_type",
-              "expr": "$.contentdata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "content_type",
-              "expr": "$.contentdata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "content_media_type",
-              "expr": "$.contentdata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "content_language",
-              "expr": "$.contentdata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_medium",
-              "expr": "$.contentdata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_gradelevel",
-              "expr": "$.contentdata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_subjects",
-              "expr": "$.contentdata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "content_mimetype",
-              "expr": "$.contentdata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "content_framework",
-              "expr": "$.contentdata.framework"
-            },
-            {
-              "type": "path",
-              "name": "content_board",
-              "expr": "$.contentdata.board"
-            },
-            {
-              "type": "path",
-              "name": "content_status",
-              "expr": "$.contentdata.status"
-            },
-            {
-              "type": "path",
-              "name": "content_version",
-              "expr": "$.contentdata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "content_last_submitted_on",
-              "expr": "$.contentdata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_published_on",
-              "expr": "$.contentdata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "content_last_updated_on",
-              "expr": "$.contentdata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "content_created_by",
-              "expr": "$.contentdata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "content_created_for",
-              "expr": "$.contentdata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "collection_name",
-              "expr": "$.collectiondata.name"
-            },
-            {
-              "type": "path",
-              "name": "collection_object_type",
-              "expr": "$.collectiondata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_type",
-              "expr": "$.collectiondata.contenttype"
-            },
-            {
-              "type": "path",
-              "name": "collection_media_type",
-              "expr": "$.collectiondata.mediatype"
-            },
-            {
-              "type": "path",
-              "name": "collection_language",
-              "expr": "$.collectiondata.language[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_medium",
-              "expr": "$.collectiondata.medium[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_gradelevel",
-              "expr": "$.collectiondata.gradelevel[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_subjects",
-              "expr": "$.collectiondata.subject[*]"
-            },
-            {
-              "type": "path",
-              "name": "collection_mimetype",
-              "expr": "$.collectiondata.mimetype"
-            },
-            {
-              "type": "path",
-              "name": "collection_framework",
-              "expr": "$.collectiondata.framework"
-            },
-            {
-              "type": "path",
-              "name": "collection_board",
-              "expr": "$.collectiondata.board"
-            },
-            {
-              "type": "path",
-              "name": "collection_status",
-              "expr": "$.collectiondata.status"
-            },
-            {
-              "type": "path",
-              "name": "collection_version",
-              "expr": "$.collectiondata.pkgversion"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_submitted_on",
-              "expr": "$.collectiondata.lastsubmittedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_published_on",
-              "expr": "$.collectiondata.lastpublishedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_last_updated_on",
-              "expr": "$.collectiondata.lastupdatedon"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_by",
-              "expr": "$.collectiondata.createdby"
-            },
-            {
-              "type": "path",
-              "name": "collection_created_for",
-              "expr": "$.collectiondata.createdfor"
-            },
-            {
-              "type": "path",
-              "name": "user_grade_list",
-              "expr": "$.userdata.gradelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_language_list",
-              "expr": "$.userdata.languagelist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_subject_list",
-              "expr": "$.userdata.subjectlist[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_type",
-              "expr": "$.userdata.usertype"
-            },
-            {
-              "type": "path",
-              "name": "user_roles",
-              "expr": "$.userdata.roles[*]"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_state",
-              "expr": "$.userdata.state"
-            },
-            {
-              "type": "path",
-              "name": "user_loc_district",
-              "expr": "$.userdata.district"
-            },
-            {
-              "type": "path",
-              "name": "user_signin_type",
-              "expr": "$.userdata.usersignintype"
-            },
-            {
-              "type": "path",
-              "name": "user_login_type",
-              "expr": "$.userdata.userlogintype"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_channel",
-              "expr": "$.dialcodedata.channel"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_batchcode",
-              "expr": "$.dialcodedata.batchcode"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_publisher",
-              "expr": "$.dialcodedata.publisher"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_generated_on",
-              "expr": "$.dialcodedata.generatedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_published_on",
-              "expr": "$.dialcodedata.publishedon"
-            },
-            {
-              "type": "path",
-              "name": "dialcode_object_type",
-              "expr": "$.dialcodedata.objecttype"
-            },
-            {
-              "type": "path",
-              "name": "device_loc_iso_state_code",
-              "expr": "$.devicedata.iso3166statecode"
-            }
-          ]
-        },
-        "dimensionsSpec": {
-          "dimensions": [
-            {
-              "type": "string",
-              "name": "eid"
-            },
-            {
-              "type": "string",
-              "name": "mid"
-            },
-            {
-              "type": "long",
-              "name": "ets"
-            },
-            {
-              "type": "string",
-              "name": "@timestamp"
-            },
-            {
-              "type": "string",
-              "name": "actor_id"
-            },
-            {
-              "type": "string",
-              "name": "actor_type"
-            },
-            {
-              "type": "string",
-              "name": "context_channel"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_pid"
-            },
-            {
-              "type": "string",
-              "name": "context_pdata_ver"
-            },
-            {
-              "type": "string",
-              "name": "context_env"
-            },
-            {
-              "type": "string",
-              "name": "context_sid"
-            },
-            {
-              "type": "string",
-              "name": "context_did"
-            },
-            {
-              "name": "context_cdata_type"
-            },
-            {
-              "name": "context_cdata_id"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "context_rollup_l4"
-            },
-            {
-              "type": "string",
-              "name": "object_id"
-            },
-            {
-              "type": "string",
-              "name": "object_type"
-            },
-            {
-              "type": "string",
-              "name": "object_version"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l1"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l2"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l3"
-            },
-            {
-              "type": "string",
-              "name": "object_rollup_l4"
-            },
-            {
-              "name": "tags"
-            },
-            {
-              "type": "string",
-              "name": "edata_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_subtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_mode"
-            },
-            {
-              "type": "string",
-              "name": "edata_pageid"
-            },
-            {
-              "type": "string",
-              "name": "edata_uri"
-            },
-            {
-              "type": "string",
-              "name": "edata_id"
-            },
-            {
-              "type": "double",
-              "name": "edata_duration"
-            },
-            {
-              "type": "long",
-              "name": "edata_index"
-            },
-            {
-              "type": "string",
-              "name": "edata_pass"
-            },
-            {
-              "type": "double",
-              "name": "edata_score"
-            },
-            {
-              "name": "edata_resvalues"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_item_title"
-            },
-            {
-              "type": "double",
-              "name": "edata_item_maxscore"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_target_type"
-            },
-            {
-              "type": "long",
-              "name": "edata_rating"
-            },
-            {
-              "type": "string",
-              "name": "edata_comments"
-            },
-            {
-              "type": "string",
-              "name": "edata_commentid"
-            },
-            {
-              "type": "string",
-              "name": "edata_commenttxt"
-            },
-            {
-              "type": "string",
-              "name": "edata_dir"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_origin_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_items_to_type"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_id"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_ver"
-            },
-            {
-              "type": "string",
-              "name": "edata_plugin_category"
-            },
-            {
-              "name": "edata_props"
-            },
-            {
-              "type": "string",
-              "name": "edata_state"
-            },
-            {
-              "type": "string",
-              "name": "edata_prevstate"
-            },
-            {
-              "type": "long",
-              "name": "edata_size"
-            },
-            {
-              "name": "edata_filters_dialcodes"
-            },
-            {
-              "type": "string",
-              "name": "edata_topn_identifier"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objid"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objtype"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_objver"
-            },
-            {
-              "type": "string",
-              "name": "edata_visits_index"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_city"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_country"
-            },
-            {
-              "type": "string",
-              "name": "device_os"
-            },
-            {
-              "type": "string",
-              "name": "device_make"
-            },
-            {
-              "type": "string",
-              "name": "device_id"
-            },
-            {
-              "type": "long",
-              "name": "device_mem"
-            },
-            {
-              "type": "string",
-              "name": "device_idisk"
-            },
-            {
-              "type": "string",
-              "name": "device_edisk"
-            },
-            {
-              "type": "string",
-              "name": "device_scrn"
-            },
-            {
-              "type": "string",
-              "name": "device_camera"
-            },
-            {
-              "type": "string",
-              "name": "device_cpu"
-            },
-            {
-              "type": "long",
-              "name": "device_sims"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_agent"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_ver"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_system"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_platform"
-            },
-            {
-              "type": "string",
-              "name": "device_uaspec_raw"
-            },
-            {
-              "type": "long",
-              "name": "device_first_access"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_code"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_state_custom_name"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_state"
-            },
-            {
-              "type": "string",
-              "name": "user_declared_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "derived_loc_from"
-            },
-            {
-              "type": "string",
-              "name": "content_name"
-            },
-            {
-              "type": "string",
-              "name": "content_object_type"
-            },
-            {
-              "type": "string",
-              "name": "content_type"
-            },
-            {
-              "type": "string",
-              "name": "content_media_type"
-            },
-            {
-              "name": "content_language"
-            },
-            {
-              "name": "content_medium"
-            },
-            {
-              "name": "content_gradelevel"
-            },
-            {
-              "name": "content_subjects"
-            },
-            {
-              "type": "string",
-              "name": "content_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "content_framework"
-            },
-            {
-              "type": "string",
-              "name": "content_board"
-            },
-            {
-              "type": "string",
-              "name": "content_status"
-            },
-            {
-              "type": "double",
-              "name": "content_version"
-            },
-            {
-              "type": "long",
-              "name": "content_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "content_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "content_created_by"
-            },
-            {
-              "name": "content_created_for"
-            },
-            {
-              "type": "string",
-              "name": "collection_name"
-            },
-            {
-              "type": "string",
-              "name": "collection_object_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_type"
-            },
-            {
-              "type": "string",
-              "name": "collection_media_type"
-            },
-            {
-              "name": "collection_language"
-            },
-            {
-              "name": "collection_medium"
-            },
-            {
-              "name": "collection_gradelevel"
-            },
-            {
-              "name": "collection_subjects"
-            },
-            {
-              "type": "string",
-              "name": "collection_mimetype"
-            },
-            {
-              "type": "string",
-              "name": "collection_framework"
-            },
-            {
-              "type": "string",
-              "name": "collection_board"
-            },
-            {
-              "type": "string",
-              "name": "collection_status"
-            },
-            {
-              "type": "double",
-              "name": "collection_version"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_submitted_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_published_on"
-            },
-            {
-              "type": "long",
-              "name": "collection_last_updated_on"
-            },
-            {
-              "type": "string",
-              "name": "collection_created_by"
-            },
-            {
-              "name": "collection_created_for"
-            },
-            {
-              "name": "user_grade_list"
-            },
-            {
-              "name": "user_language_list"
-            },
-            {
-              "name": "user_subject_list"
-            },
-            {
-              "type": "string",
-              "name": "user_type"
-            },
-            {
-              "name": "user_roles"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_state"
-            },
-            {
-              "type": "string",
-              "name": "user_loc_district"
-            },
-            {
-              "type": "string",
-              "name": "user_signin_type"
-            },
-            {
-              "type": "string",
-              "name": "user_login_type"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_channel"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_batchcode"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_publisher"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_generated_on"
-            },
-            {
-              "type": "long",
-              "name": "dialcode_published_on"
-            },
-            {
-              "type": "string",
-              "name": "dialcode_object_type"
-            },
-            {
-              "type": "string",
-              "name": "device_loc_iso_state_code"
-            }
-          ],
-          "dimensionsExclusions": []
-        },
-        "timestampSpec": {
-          "column": "syncts",
-          "format": "auto"
-        }
-      }
-    },
-    "metricsSpec": [],
-    "granularitySpec": {
-      "type": "uniform",
-      "segmentGranularity": "day",
-      "queryGranularity": "none",
-      "rollup": false
-    }
-  },
-  "ioConfig": {
-    "topic": "{{env}}.events.telemetry",
-    "consumerProperties": {
-      "bootstrap.servers": "{{kafka_brokers}}"
-    },
-    "taskCount": 1,
-    "replicas": 1,
-    "taskDuration": "PT14400S",
-    "useEarliestOffset": false,
-    "completionTimeout" : "PT2H"
-  },
-  "tuningConfig": {
-    "type": "kafka",
-    "reportParseExceptions": false
-  }
-}
diff --git a/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml b/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml
new file mode 100644
index 0000000000..549881ea6d
--- /dev/null
+++ b/ansible/roles/druid-summary-monthly-ingestion/defaults/main.yml
@@ -0,0 +1,14 @@
+remote_user: druid
+druid_overlord_host: "{{ groups['rollup-overlord'][0] }}"
+druid_overlord_port: 8090
+druid_home: /home/druid
+
+summary_monthly:
+  base_data_source_name: "summary-rollup-syncts"
+  new_data_source_name: "summary-monthly-rollup-syncts"
+  no_batch_parallel_tasks: 1
+
+supervisor_summary_spec_url: "http://{{ druid_overlord_host }}:{{ druid_overlord_port }}/druid/indexer/v1/supervisor/{{ summary_monthly.base_data_source_name }}"
+
+druid_schema:
+ - summary_monthly_batch
diff --git a/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml b/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml
new file mode 100644
index 0000000000..908e7bc245
--- /dev/null
+++ b/ansible/roles/druid-summary-monthly-ingestion/tasks/main.yml
@@ -0,0 +1,31 @@
+- name: Get the summary supervisor spec
+  uri:
+    url: "{{ supervisor_summary_spec_url }}"
+    method: GET
+    return_content: yes
+  register : supervisor_spec
+
+- set_fact:
+   dimensions_spec: "{{ supervisor_spec.json.dataSchema.parser.parseSpec.dimensionsSpec | to_json }}"
+
+- name : Retrive Interval Month Range
+  shell : echo $(date -d"1 $(date -d'last month' +%b)" +%F)/$(date -d"-0 day 1 $(date +%b)" +%F )
+  register : output
+
+- set_fact:
+   summary_monthly_interval: "{{ output.stdout }}"
+
+- name: Copy the schema file
+  template: src="{{item}}" dest="{{druid_home}}/{{item}}.json"
+  become_user: "{{remote_user}}" 
+  with_items: "{{druid_schema}}"
+
+- name: Start batch ingestion task
+  shell: 'curl -X POST -H "Content-Type:application/json" -d @{{druid_home}}/{{item}}.json http://{{druid_overlord_host}}:{{druid_overlord_port}}/druid/indexer/v1/task'
+  with_items: "{{druid_schema}}"
+  become_user: "{{remote_user}}"
+
+
+
+
+
diff --git a/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch b/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch
new file mode 100644
index 0000000000..e2049e23f4
--- /dev/null
+++ b/ansible/roles/druid-summary-monthly-ingestion/templates/summary_monthly_batch
@@ -0,0 +1,57 @@
+{
+  "type": "index_parallel",
+  "spec": {
+    "ioConfig": {
+      "type": "index_parallel",
+      "inputSource": {
+        "type": "druid",
+        "dataSource": "{{ summary_monthly.base_data_source_name }}",
+        "interval": "{{ summary_monthly_interval }}",
+        "metrics": [
+          "total_count",
+          "total_interactions",
+          "total_timespent"
+        ]
+      },
+      "appendToExisting": false
+    },
+    "tuningConfig": {
+      "type": "index_parallel",
+      "targetPartitionSize": 5000000,
+      "maxRowsInMemory": 500000,
+      "forceExtendableShardSpecs": true,
+      "maxNumConcurrentSubTasks": "{{ summary_monthly.no_batch_parallel_tasks }}"
+    },
+    "dataSchema": {
+      "dataSource": "{{ summary_monthly.new_data_source_name }}",
+      "dimensionsSpec": {{ dimensions_spec }},
+      "timestampSpec": {
+        "column": "syncts",
+        "format": "auto"
+      },
+      "metricsSpec": [
+        {
+          "type": "longSum",
+          "name": "total_count",
+          "fieldName": "total_count"
+        },
+        {
+          "type": "longSum",
+          "name": "total_interactions",
+          "fieldName": "total_interactions"
+        },
+        {
+          "type": "doubleSum",
+          "name": "total_time_spent",
+          "fieldName": "total_time_spent"
+        }
+      ]
+      "granularitySpec": {
+        "type": "uniform",
+        "segmentGranularity": "month",
+        "queryGranularity": "none",
+        "rollup": true
+      }
+    }
+  }
+}
diff --git a/ansible/roles/ep_logstash/defaults/main.yml b/ansible/roles/ep_logstash/defaults/main.yml
index 0df77cdccc..ad18f4e8d7 100644
--- a/ansible/roles/ep_logstash/defaults/main.yml
+++ b/ansible/roles/ep_logstash/defaults/main.yml
@@ -11,10 +11,13 @@ logstash_conf:
 es_mapping_file:
  - "telemetry_mapping.json" 
  - "crash-log-mapping.json"
+ - "backend_telemetry_mapping.json"
 kafka_topic_prefix: "{{ env }}"
 error_es_index: "telemetry-error-events"
 log_es_index: "telemetry-log-events"
 telemetry_mapping_file_path: "{{ logstash_home }}/mappings/telemetry_mapping.json"
+backend_log_es_index: "dp-backend-events"
+backend_telemetry_mapping_file_path: "{{ logstash_home }}/mappings/backend_telemetry_mapping.json"
 apt_cache_valid_time: 3600
 
 zookeepers: "{{ groups['processing-cluster-zookeepers']|join(':2181,') }}:2181"
@@ -31,15 +34,10 @@ private_exhaust_topic: "telemetry.private_exhaust"
 public_exhaust_topic: "telemetry.public_exhaust"
 public_data_exhaust_dir_prefix: 4208ab995984d222b59299e5103d350a842d8d41
 
-#Device register API metrics
-device_register_api_metrics_log_path: "/mount/data/analytics/logs/api-service/api-metrics.log"
-api_job_log_path: "/mount/data/analytics/logs/api-service/joblog.log"
-device_log_path: "/mount/data/analytics/logs/api-service/device-profile.log"
-
 # client API error log
-crash_log_path: "/mount/data/analytics/logs/api-service/crash-logs.log"
 logger_es: "{{ groups['log-es']|join(':9200,') }}"
 logstash_version: '6.x'
 crash_log_mapping_file_path: "{{ logstash_home }}/mappings/crash-log-mapping.json"
 crash_log_index : "crash-logs"
 
+send_logs_to_graylog: false
diff --git a/ansible/roles/ep_logstash/tasks/common.yml b/ansible/roles/ep_logstash/tasks/common.yml
index 33cc5c022f..6ab285a8c5 100644
--- a/ansible/roles/ep_logstash/tasks/common.yml
+++ b/ansible/roles/ep_logstash/tasks/common.yml
@@ -74,11 +74,11 @@
   become: yes
   ignore_errors: yes
 
-
 - name: Copy logstash init.d file
   copy: 
     src: logstash.sh 
-    dest: /etc/init.d/logstash mode=755
+    dest: /etc/init.d/logstash 
+    mode: 755
   become: true
 
 - name: Install plugins
diff --git a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics
deleted file mode 100644
index aea3fa75f2..0000000000
--- a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.device_api_metrics
+++ /dev/null
@@ -1,69 +0,0 @@
-input {
-    file {
-        path => "{{device_register_api_metrics_log_path}}"
-        start_position => "end"
-        sincedb_write_interval => 10
-        type => "device_register_api_metrics"
-    }
-    file {
-        path => "{{api_job_log_path}}"
-        start_position => "end"
-        sincedb_write_interval => 10
-        type => "api_job_logs"
-    }
-    file {
-        path => "{{crash_log_path}}"
-        codec => "json"
-        start_position => "end"
-        sincedb_write_interval => 10
-        type => "crash_logs"
-    }
-    file {
-        path => "{{device_log_path}}"
-        start_position => "end"
-        sincedb_write_interval => 10
-        type => "device_logs"
-    }
-}
-output {
-  if [type] == "device_logs" {
-      kafka {
-        bootstrap_servers => "{{kafka_brokers}}"
-        topic_id => "{{kafka_topic_prefix}}.events.deviceprofile"
-        codec => plain {
-          format => "%{message}"
-        }
-      }
-  }
-
-  if [type] == "device_register_api_metrics" {
-    kafka {
-      bootstrap_servers => "{{kafka_brokers}}"
-      topic_id => "{{kafka_topic_prefix}}.pipeline_metrics"
-      codec => line {
-        format => "%{message}"
-      }
-    }
-  } 
-
-  if [type] == "api_job_logs" {
-      kafka {
-        bootstrap_servers => "{{kafka_brokers}}"
-        topic_id => "{{kafka_topic_prefix}}.telemetry.log"
-        codec => line {
-          format => "%{message}"
-        }
-      }
-  }
-
-  if [type] == "crash_logs" {
-      elasticsearch {
-        hosts => ["{{logger_es}}"]
-        template => "{{crash_log_mapping_file_path}}"
-        template_name => "crash-log"
-        index => "{{crash_log_index}}"
-        manage_template => true
-        document_type => "logs"
-      }
-  }
-}
diff --git a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events
index 949f3f68de..b1fa9718f3 100644
--- a/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events
+++ b/ansible/roles/ep_logstash/templates/logstash.conf.indexer.log_error.events
@@ -1,11 +1,12 @@
+{% if not send_logs_to_graylog %}
 # Logstash configuration 
 # Kafka(telemetry.log) -> Logstash -> Elasticsearch pipeline index error type events into(telemetry-error-events) and log type events into(telemetry-log-index).
 
 input {  
 	kafka {
 		bootstrap_servers => "{{bootstrap_server}}"
-		topics => ["{{kafka_topic_prefix}}.telemetry.log"]
-		group_id => "{{kafka_topic_prefix}}.telemetry.log"
+		topics => ["{{kafka_topic_prefix}}.druid.events.log", "{{kafka_topic_prefix}}.druid.events.error"]
+		group_id => "{{kafka_topic_prefix}}.telemetry-log.indexer"
 		auto_offset_reset => "latest"
 		codec => "json"
 		consumer_threads => 1
@@ -34,9 +35,16 @@ output {
 			manage_template => true        
 			document_type => "events"
 		}
+	}else if "JOB_" in [eid] {
+		elasticsearch {
+			codec => "json"
+			hosts => ["{{logger_es}}"]
+			template => "{{backend_telemetry_mapping_file_path}}"
+			template_name => "dp-backend"
+			index => "{{backend_log_es_index}}-%{+YYYY.MM.ww}"
+			manage_template => true        
+			document_type => "events"
+		}
 	}
 }
-
-
-
-
+{% endif %}
\ No newline at end of file
diff --git a/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json b/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json
new file mode 100644
index 0000000000..e376c0d6b0
--- /dev/null
+++ b/ansible/roles/ep_logstash/templates/mapping/backend_telemetry_mapping.json
@@ -0,0 +1,138 @@
+{
+  "template" : "backend",
+  "index_patterns" : "backend-*",
+  "settings" : {
+    "number_of_shards" : 5
+  },
+  "mappings" : {
+    "events" : {
+      "dynamic": false,
+      "properties": {
+          "@timestamp": {
+            "format": "strict_date_optional_time||epoch_millis",
+            "type": "date"
+          },
+          "@version": {
+            "type": "keyword"
+          },
+          "eid": {
+            "type": "keyword"
+          },
+          "ets": {
+            "format": "strict_date_optional_time||epoch_millis",
+            "type": "date"
+          },
+          "mid": {
+            "type": "keyword"
+          },
+          "ts": {
+            "format": "strict_date_optional_time||epoch_millis",
+            "type": "date"
+          },
+          "ver": {
+            "type": "keyword"
+          },
+          "actor": {
+             "properties": {
+                "id": {
+                  "type": "keyword"
+                },
+                "type": {
+                  "type": "keyword"
+                }
+             }
+          },
+          "edata": {
+            "properties": {
+              "data": {
+                "type": "object",
+                "properties": {
+                  "config": {
+                    "properties": {
+                      "search": {
+                        "type": "object",
+                      },
+                      "model": {
+                        "type": "keyword"
+                      },
+                      "modelParams": {
+                        "type": "object",
+                      },
+                      "output": {
+                        "type": "object"
+                      },
+                      "parallelization": {
+                        "type": "long"
+                      },
+                      "appName": {
+                        "type": "keyword"
+                      },
+                      "deviceMapping": {
+                        "type": "boolean"
+                      }
+                    }
+                  },
+                  "model": {
+                    "type": "keyword"
+                  },
+                  "date": {
+                    "type": "keyword"
+                  },
+                  "inputEvents": {
+                    "type": "long"
+                  },
+                  "outputEvents": {
+                    "type": "long"
+                  },
+                  "timeTaken": {
+                    "type": "double"
+                  }
+                }
+              },
+              "level": {
+                "type": "keyword"
+              },
+              "message": {
+                "type": "keyword"
+              },
+              "class": {
+                "type": "keyword"
+              },
+              "status": {
+                "type": "keyword"
+              }
+            }
+          },
+          "context": {
+            "properties": {
+              "channel": {
+                "type": "keyword"
+              },
+              "env": {
+                "type": "keyword"
+              },
+              "sid": {
+                "type": "keyword"
+              },
+              "did": {
+                "type": "keyword"
+              },
+              "pdata": {
+                "properties": {
+                  "id": {
+                    "type": "keyword"
+                  },
+                  "pid": {
+                    "type": "keyword"
+                  },
+                  "ver": {
+                    "type": "keyword"
+                  }
+                }
+              }
+            }
+          }
+       }
+    }
+  }
+}
\ No newline at end of file
diff --git a/ansible/roles/es-azure-restore/defaults/main.yml b/ansible/roles/es-azure-restore/defaults/main.yml
index d5946184d1..2f70c0d69d 100644
--- a/ansible/roles/es-azure-restore/defaults/main.yml
+++ b/ansible/roles/es-azure-restore/defaults/main.yml
@@ -1,4 +1,6 @@
 es_restore_host: localhost
+snapshot_base_path: application
+
 snapshot_create_request_body: {
     type: azure,
     settings: {
@@ -6,4 +8,5 @@ snapshot_create_request_body: {
       base_path: "{{ snapshot_base_path }}"
     }
 }
-snapshot_base_path: folderNameInTheAzureContainer 
+
+#snapshot_base_path: folderNameInTheAzureContainer 
diff --git a/ansible/roles/es-azure-restore/tasks/main.yml b/ansible/roles/es-azure-restore/tasks/main.yml
index 6ebc6e7197..ff71c19a11 100644
--- a/ansible/roles/es-azure-restore/tasks/main.yml
+++ b/ansible/roles/es-azure-restore/tasks/main.yml
@@ -1,16 +1,7 @@
 ---
-#- name: Register with azure blob 
-#  uri:
-#    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup"
-#    method: PUT
-#    headers:
-#      Content-Type: "application/json"
-#    body_format : json
-#    data: 
-
-- name: Create azure snapshot
+- name: Set azure snapshot for the first time
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT
     body: "{{ snapshot_create_request_body | to_json }}"
     headers:
@@ -18,17 +9,17 @@
 
 - name: Restore ES from Azure backup
   uri:
-    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_restore"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_restore"
     method: POST
 
 - name: "Wait for restore to be completed"
   uri:
-    url: "http://{{ es_restore_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}/_status"
+    url: "http://{{ es_restore_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}/_status"
     method: GET
     return_content: yes
     status_code: 200
     body_format: json
   register: result
   until: result.json.snapshots[0].state == 'SUCCESS'
-  retries: 360
+  retries: 1200
   delay: 10
diff --git a/ansible/roles/es-azure-snapshot/defaults/main.yml b/ansible/roles/es-azure-snapshot/defaults/main.yml
index 14b39c8539..de88d89398 100644
--- a/ansible/roles/es-azure-snapshot/defaults/main.yml
+++ b/ansible/roles/es-azure-snapshot/defaults/main.yml
@@ -1,7 +1,15 @@
 snapshot_create_request_body: {
     type: azure,
     settings: {
-      container: "elasticsearch-snapshots",
-      base_path: "{{ snapshot_base_path }}"
+      container: "{{ cloud_storage_esbackup_foldername }}",
+      base_path: "{{ snapshot_base_path }}_{{ base_path_date }}"
     }
 }
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+
+cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_esbackup_foldername: "elasticsearch-snapshots"
+
diff --git a/ansible/roles/es-azure-snapshot/tasks/main.yml b/ansible/roles/es-azure-snapshot/tasks/main.yml
index a39b963590..23be535db9 100644
--- a/ansible/roles/es-azure-snapshot/tasks/main.yml
+++ b/ansible/roles/es-azure-snapshot/tasks/main.yml
@@ -1,43 +1,54 @@
 ---
-- name: Create azure snapshot
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: create container in azure storage if it doesn't exist
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: container-create.yml
+    apply:
+      delegate_to: localhost
+  vars:
+    blob_container_name: "{{ es_backup_storage }}"
+    container_public_access: "off"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+
+- name: Create Azure Repository
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
     method: PUT
     body: "{{ snapshot_create_request_body | to_json }}"
     headers:
       Content-Type: "application/json"
 
-- set_fact: snapshot_number="snapshot_{{ansible_date_time.epoch}}"
-
 - name: Take new snapshot
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: PUT
-    body:  >
-      {"indices":"*","include_global_state":false}
     headers:
       Content-Type: "application/json"
-  async: 45000
-  poll: 10
 
 - name: Print all snapshots
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/_all"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
     method: GET
 
 - name: Print status of current snapshot
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: GET
 
 - name: "Wait for backup to be completed"
   uri:
-    url: "http://{{ es_snapshot_host }}:9200/_snapshot/azurebackup/{{snapshot_number}}"
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
     method: GET
     return_content: yes
     status_code: 200
     body_format: json
   register: result
   until: result.json.snapshots[0].state == 'SUCCESS'
-  retries: 300
+  retries: 120
   delay: 10
diff --git a/ansible/roles/es-curator/defaults/main.yml b/ansible/roles/es-curator/defaults/main.yml
new file mode 100644
index 0000000000..9fd4efe2c8
--- /dev/null
+++ b/ansible/roles/es-curator/defaults/main.yml
@@ -0,0 +1,2 @@
+es_curator_major_version: 5
+es_curator_version: 5.8.4
\ No newline at end of file
diff --git a/ansible/roles/es-curator/tasks/main.yml b/ansible/roles/es-curator/tasks/main.yml
new file mode 100644
index 0000000000..c4a8bacee7
--- /dev/null
+++ b/ansible/roles/es-curator/tasks/main.yml
@@ -0,0 +1,13 @@
+- name: Debian - Add Elasticsearch repository key
+  apt_key: url="https://artifacts.elastic.co/GPG-KEY-elasticsearch" state=present
+
+- name: Add curator {{ es_curator_major_version }} repo
+  apt_repository: repo='deb [arch=amd64] http://packages.elastic.co/curator/{{ es_curator_major_version }}/debian stable main' state=present update_cache=yes
+
+- debug:
+    msg: "{{ es_curator_version }}"
+
+- name: Install elasticsearch curator
+  apt:
+    name: elasticsearch-curator={{ es_curator_version }}
+    force: yes
diff --git a/ansible/roles/es-gcs-snapshot/defaults/main.yml b/ansible/roles/es-gcs-snapshot/defaults/main.yml
new file mode 100644
index 0000000000..7222b0c06b
--- /dev/null
+++ b/ansible/roles/es-gcs-snapshot/defaults/main.yml
@@ -0,0 +1,14 @@
+snapshot_create_request_body: {
+    type: gcs,
+    settings: {
+      bucket: "{{ cloud_storage_management_bucketname }}",
+      base_path: "{{ cloud_storage_esbackup_foldername }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+    }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+
+cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_esbackup_foldername: "elasticsearch-snapshots"
diff --git a/ansible/roles/es-gcs-snapshot/tasks/main.yml b/ansible/roles/es-gcs-snapshot/tasks/main.yml
new file mode 100644
index 0000000000..55f50b17ad
--- /dev/null
+++ b/ansible/roles/es-gcs-snapshot/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create GCS Repository
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
+
+- name: Take new snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: PUT
+    headers:
+      Content-Type: "application/json"
+
+- name: Print all snapshots
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+    method: GET
+
+- name: Print status of current snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+
+- name: "Wait for backup to be completed"
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+    return_content: yes
+    status_code: 200
+    body_format: json
+  register: result
+  until: result.json.snapshots[0].state == 'SUCCESS'
+  retries: 120
+  delay: 10
diff --git a/ansible/roles/es-s3-snapshot/defaults/main.yml b/ansible/roles/es-s3-snapshot/defaults/main.yml
new file mode 100644
index 0000000000..316ae512fb
--- /dev/null
+++ b/ansible/roles/es-s3-snapshot/defaults/main.yml
@@ -0,0 +1,14 @@
+snapshot_create_request_body: {
+    type: s3,
+    settings: {
+      bucket: "{{ cloud_storage_esbackup_bucketname }}",
+      base_path: "{{ cloud_storage_esbackup_foldername }}/{{ snapshot_base_path }}_{{ base_path_date }}"
+    }
+}
+
+# Override these values
+es_snapshot_host: "localhost"
+snapshot_base_path: "default"
+
+cloud_storage_esbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_esbackup_foldername: "elasticsearch-snapshots"
diff --git a/ansible/roles/es-s3-snapshot/tasks/main.yml b/ansible/roles/es-s3-snapshot/tasks/main.yml
new file mode 100644
index 0000000000..aee768626c
--- /dev/null
+++ b/ansible/roles/es-s3-snapshot/tasks/main.yml
@@ -0,0 +1,42 @@
+---
+
+- set_fact: base_path_date="{{ lookup('pipe','date +%Y-%m') }}"
+
+- set_fact: snapshot_number="snapshot_{{ lookup('pipe','date +%s') }}"
+
+- name: Create S3 Repository
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}"
+    method: PUT
+    body: "{{ snapshot_create_request_body | to_json }}"
+    headers:
+      Content-Type: "application/json"
+
+- name: Take new snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: PUT
+    headers:
+      Content-Type: "application/json"
+
+- name: Print all snapshots
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/_all"
+    method: GET
+
+- name: Print status of current snapshot
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+
+- name: "Wait for backup to be completed"
+  uri:
+    url: "http://{{ es_snapshot_host }}:9200/_snapshot/{{ snapshot_base_path }}/{{ snapshot_number }}"
+    method: GET
+    return_content: yes
+    status_code: 200
+    body_format: json
+  register: result
+  until: result.json.snapshots[0].state == 'SUCCESS'
+  retries: 120
+  delay: 10
diff --git a/ansible/roles/es5-snapshot-purge/defaults/main.yml b/ansible/roles/es5-snapshot-purge/defaults/main.yml
new file mode 100644
index 0000000000..cab1182a3d
--- /dev/null
+++ b/ansible/roles/es5-snapshot-purge/defaults/main.yml
@@ -0,0 +1,5 @@
+es_snapshot_host: localhost
+es_snapshot_repository: "{{ snapshot_base_path }}"
+es_snapshot_retention_days: 30
+es_curator_config_dir: /etc/curator
+es_curator_config_file: "{{ es_curator_config_dir }}/curator.yml"
diff --git a/ansible/roles/es5-snapshot-purge/meta/main.yml b/ansible/roles/es5-snapshot-purge/meta/main.yml
new file mode 100644
index 0000000000..8b4e268b5d
--- /dev/null
+++ b/ansible/roles/es5-snapshot-purge/meta/main.yml
@@ -0,0 +1,3 @@
+---
+dependencies:
+  - { role: es-curator, es_curator_major_version: 5, es_curator_version: 5.8.4 }
diff --git a/ansible/roles/es5-snapshot-purge/tasks/main.yml b/ansible/roles/es5-snapshot-purge/tasks/main.yml
new file mode 100644
index 0000000000..21364440ce
--- /dev/null
+++ b/ansible/roles/es5-snapshot-purge/tasks/main.yml
@@ -0,0 +1,14 @@
+# See meta folder for curator installation
+- name: Ensure curator config dir exists
+  file: dest="{{es_curator_config_dir}}" state=directory
+
+- name: Create curator.yml
+  template: src=curator.yml dest="{{es_curator_config_file}}"
+
+- name: Create snapshot-purge-action.yml
+  template: src=snapshot-purge-action.yml dest="{{es_curator_config_dir}}/snapshot-purge-action.yml"
+
+- name: Delete snapshots older than {{ es_snapshot_retention_days }} days
+  shell: "curator --config {{ es_curator_config_file }} {{es_curator_config_dir}}/snapshot-purge-action.yml"
+  async: 800
+  poll: 20
diff --git a/ansible/roles/es5-snapshot-purge/templates/curator.yml b/ansible/roles/es5-snapshot-purge/templates/curator.yml
new file mode 100644
index 0000000000..6c9c6c9976
--- /dev/null
+++ b/ansible/roles/es5-snapshot-purge/templates/curator.yml
@@ -0,0 +1,10 @@
+client:
+  hosts:
+    - {{ es_snapshot_host }}
+  port: 9200
+  timeout: 100
+logging:
+    loglevel: INFO
+    logfile:
+    logformat: default
+    blacklist: ['elasticsearch', 'urllib3']
\ No newline at end of file
diff --git a/ansible/roles/es5-snapshot-purge/templates/snapshot-purge-action.yml b/ansible/roles/es5-snapshot-purge/templates/snapshot-purge-action.yml
new file mode 100644
index 0000000000..bd6cc05b06
--- /dev/null
+++ b/ansible/roles/es5-snapshot-purge/templates/snapshot-purge-action.yml
@@ -0,0 +1,13 @@
+actions:
+  1:
+    action: delete_snapshots
+    description: Delete snapshots older than {{ es_snapshot_retention_days }} days
+    options:
+      repository: {{ es_snapshot_repository }}
+      ignore_empty_list: True
+    filters:
+    - filtertype: age
+      source: creation_date
+      direction: older
+      unit: days
+      unit_count: {{ es_snapshot_retention_days }}
diff --git a/ansible/roles/es6/LICENSE b/ansible/roles/es6/LICENSE
deleted file mode 100644
index 0455cacdf6..0000000000
--- a/ansible/roles/es6/LICENSE
+++ /dev/null
@@ -1,13 +0,0 @@
-Copyright (c) 2012-2016 Elasticsearch <http://www.elastic.co>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
diff --git a/ansible/roles/es6/README.md b/ansible/roles/es6/README.md
deleted file mode 100644
index 85f97acf43..0000000000
--- a/ansible/roles/es6/README.md
+++ /dev/null
@@ -1,415 +0,0 @@
-# ansible-elasticsearch
-[![Ansible Galaxy](https://img.shields.io/badge/ansible--galaxy-elastic.elasticsearch-blue.svg)](https://galaxy.ansible.com/elastic/elasticsearch/)
-
-**THIS ROLE IS FOR 6.x, 5.x. FOR 2.x SUPPORT PLEASE USE THE 2.x BRANCH.**
-
-Ansible role for 6.x/5.x Elasticsearch.  Currently this works on Debian and RedHat based linux systems.  Tested platforms are:
-
-* Ubuntu 14.04/16.04
-* Debian 8
-* Centos 7
-
-The latest Elasticsearch versions of 6.x are actively tested.  **Only Ansible versions > 2.3.2 are supported, as this is currently the only version tested.**
-
-##### Dependency
-This role uses the json_query filter which [requires jmespath](https://github.com/ansible/ansible/issues/24319) on the local machine.
-
-## Usage
-
-Create your Ansible playbook with your own tasks, and include the role elasticsearch.  You will have to have this repository accessible within the context of playbook, e.g.
-
-e.g. 
-
-```
-cd /my/repos/
-git clone https://github.com/elastic/ansible-elasticsearch.git
-cd /my/ansible/playbook
-mkdir -p roles
-ln -s /my/repos/ansible-elasticsearch ./roles/elasticsearch
-```
-
-Then create your playbook yaml adding the role elasticsearch. By default, the user is only required to specify a unique es_instance_name per role application.  This should be unique per node. 
-The application of the elasticsearch role results in the installation of a node on a host.
-
-The simplest configuration therefore consists of: 
-
-```
-- name: Simple Example
-  hosts: localhost
-  roles:
-    - { role: elasticsearch, es_instance_name: "node1" }
-  vars:
-```
-
-The above installs a single node 'node1' on the hosts 'localhost'.
-
-This role also uses [Ansible tags](http://docs.ansible.com/ansible/playbooks_tags.html). Run your playbook with the `--list-tasks` flag for more information.
-
-### Basic Elasticsearch Configuration
-
-All Elasticsearch configuration parameters are supported.  This is achieved using a configuration map parameter 'es_config' which is serialized into the elasticsearch.yml file.  
-The use of a map ensures the Ansible playbook does not need to be updated to reflect new/deprecated/plugin configuration parameters.
-
-In addition to the es_config map, several other parameters are supported for additional functions e.g. script installation.  These can be found in the role's defaults/main.yml file.
-
-The following illustrates applying configuration parameters to an Elasticsearch instance.  By default, Elasticsearch 5.1.2is installed.
-
-```
-- name: Elasticsearch with custom configuration
-  hosts: localhost
-  roles:
-    #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", 
-    es_config: {
-        node.name: "node1", 
-        cluster.name: "custom-cluster",
-        discovery.zen.ping.unicast.hosts: "localhost:9301",
-        http.port: 9201,
-        transport.tcp.port: 9301,
-        node.data: false,
-        node.master: true,
-        bootstrap.memory_lock: true,
-        } 
-    }
-  vars:
-    es_scripts: false
-    es_templates: false
-    es_version_lock: false
-    es_heap_size: 1g
-    es_api_port: 9201
-```
-
-Whilst the role installs Elasticsearch with the default configuration parameters, the following should be configured to ensure a cluster successfully forms:
-
-* ```es_config['http.port']``` - the http port for the node
-* ```es_config['transport.tcp.port']``` - the transport port for the node
-* ```es_config['discovery.zen.ping.unicast.hosts']``` - the unicast discovery list, in the comma separated format ```"<host>:<port>,<host>:<port>"``` (typically the clusters dedicated masters)
-* ```es_config['network.host']``` - sets both network.bind_host and network.publish_host to the same host value. The network.bind_host setting allows to control the host different network components will bind on.  
-
-The network.publish_host setting allows to control the host the node will publish itself within the cluster so other nodes will be able to connect to it. 
-
-See https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-network.html for further details on default binding behaviour and available options.
-The role makes no attempt to enforce the setting of these are requires users to specify them appropriately.  IT is recommended master nodes are listed and thus deployed first where possible.
-
-A more complex example:
-
-```
-- name: Elasticsearch with custom configuration
-  hosts: localhost
-  roles:
-    #expand to all available parameters
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch/data", es_log_dir: "/opt/elasticsearch/logs", 
-    es_config: {
-        node.name: "node1", 
-        cluster.name: "custom-cluster", 
-        discovery.zen.ping.unicast.hosts: "localhost:9301", 
-        http.port: 9201, 
-        transport.tcp.port: 9301, 
-        node.data: false, 
-        node.master: true, 
-        bootstrap.memory_lock: true, 
-        } 
-    }
-  vars:
-    es_scripts: false
-    es_templates: false
-    es_version_lock: false
-    es_heap_size: 1g
-    es_start_service: false
-    es_plugins_reinstall: false
-    es_api_port: 9201
-    es_plugins:
-        - plugin: ingest-geoip
-          proxy_host: proxy.example.com
-          proxy_port: 8080
-```
-
-#### Important Note
-
-**The role uses es_api_host and es_api_port to communicate with the node for actions only achievable via http e.g. to install templates and to check the NODE IS ACTIVE.  These default to "localhost" and 9200 respectively.  
-If the node is deployed to bind on either a different host or port, these must be changed.**
-
-### Multi Node Server Installations
-
-The application of the elasticsearch role results in the installation of a node on a host. Specifying the role multiple times for a host therefore results in the installation of multiple nodes for the host. 
-
-An example of a two server deployment is shown below.  The first server holds the master and is thus declared first.  Whilst not mandatory, this is recommended in any multi node cluster configuration.  The second server hosts two data nodes.
-
-**Note the structure of the below playbook for the data nodes.  Whilst a more succinct structures are possible which allow the same role to be applied to a host multiple times, we have found the below structure to be the most reliable with respect to var behaviour.  This is the tested approach.**
-
-```
-- hosts: master_nodes
-  roles:
-    - { role: elasticsearch, es_instance_name: "node1", es_heap_size: "1g",
-    es_config: {
-        cluster.name: "test-cluster",
-        discovery.zen.ping.unicast.hosts: "elastic02:9300",
-        http.port: 9200,
-        transport.tcp.port: 9300,
-        node.data: false,
-        node.master: true,
-        bootstrap.memory_lock: false,
-        }
-    }
-  vars:
-    es_scripts: false
-    es_templates: false
-    es_version_lock: false
-    ansible_user: ansible
-    es_plugins:
-     - plugin: ingest-geoip
-
-
-
-- hosts: data_nodes
-  roles:
-    - { role: elasticsearch, es_instance_name: "node1", es_data_dirs: "/opt/elasticsearch", 
-    es_config: {
-        discovery.zen.ping.unicast.hosts: "elastic02:9300",
-        http.port: 9200,
-        transport.tcp.port: 9300,
-        node.data: true,
-        node.master: false,
-        bootstrap.memory_lock: false,
-        cluster.name: "test-cluster"
-        } 
-    }
-  vars:
-    es_scripts: false
-    es_templates: false
-    es_version_lock: false
-    ansible_user: ansible
-    es_api_port: 9200
-    es_plugins:
-    - plugin: ingest-geoip
-    
-    
-- hosts: data_nodes
-  roles:
-    - { role: elasticsearch, es_instance_name: "node2", es_api_port:9201,
-    es_config: {
-        discovery.zen.ping.unicast.hosts: "elastic02:9300",
-        http.port: 9201,
-        transport.tcp.port: 9301,
-        node.data: true,
-        node.master: false,
-        bootstrap.memory_lock: false,
-        cluster.name: "test-cluster",
-        } 
-    }
-  vars:
-    es_scripts: false
-    es_templates: false
-    es_version_lock: false
-    es_api_port: 9201
-    ansible_user: ansible
-    es_plugins:
-    - plugin: ingest-geoip
-
-```
-
-Parameters can additionally be assigned to hosts using the inventory file if desired.
-
-Make sure your hosts are defined in your ```inventory``` file with the appropriate ```ansible_ssh_host```,  ```ansible_ssh_user``` and ```ansible_ssh_private_key_file``` values.
-
-Then run it:
-
-```
-ansible-playbook -i hosts ./your-playbook.yml
-```
-
-### Installing X-Pack Features
-
-X-Pack features, such as Security, are supported. This feature is currently experimental.  To enable X-Pack set the parameter `es_enable_xpack` to true and list the required features in the parameter `es_xpack_features`.  
-
-The parameter `es_xpack_features` by default enables all features i.e. it defaults to ["alerting","monitoring","graph","security","ml"]
-
-The following additional parameters allow X-Pack to be configured:
-
-* ```es_message_auth_file``` System Key field to allow message authentication. This file should be placed in the 'files' directory.
-* ```es_xpack_custom_url``` Url from which X-Pack can be downloaded. This can be used for installations in isolated environments where the elastic.co repo is not accessible. e.g. ```es_xpack_custom_url: "https://artifacts.elastic.co/downloads/packs/x-pack/x-pack-5.5.1.zip"```
-* ```es_role_mapping``` Role mappings file declared as yml as described [here](https://www.elastic.co/guide/en/x-pack/current/mapping-roles.html)
-
-
-```
-es_role_mapping:
-  power_user:
-    - "cn=admins,dc=example,dc=com"
-  user:
-    - "cn=users,dc=example,dc=com"
-    - "cn=admins,dc=example,dc=com"
-```
-
-* ```es_users``` - Users can be declared here as yml. Two sub keys 'native' and 'file' determine the realm under which realm the user is created.  Beneath each of these keys users should be declared as yml entries. e.g.
-
-```
-es_users:
-  native:
-    kibana4_server:
-      password: changeMe
-      roles:
-        - kibana4_server
-  file:
-    es_admin:
-      password: changeMe
-      roles:
-        - admin
-    testUser:
-      password: changeMeAlso!
-      roles:
-        - power_user
-        - user
-```
-            
-            
-* ```es_roles``` - Elasticsearch roles can be declared here as yml. Two sub keys 'native' and 'file' determine how the role is created i.e. either through a file or http(native) call.  Beneath each key list the roles with appropriate permissions, using the file based format described [here] (https://www.elastic.co/guide/en/x-pack/current/file-realm.html) e.g.
-
-```
-es_roles:
-  file:
-    admin:
-      cluster:
-        - all
-      indices:
-        - names: '*'
-          privileges:
-            - all
-    power_user:
-      cluster:
-        - monitor
-      indices:
-        - names: '*'
-          privileges:
-            - all
-    user:
-      indices:
-        - names: '*'
-          privileges:
-            - read
-    kibana4_server:
-      cluster:
-          - monitor
-      indices:
-        - names: '.kibana'
-          privileges:
-            - all
-  native:
-    logstash:
-      cluster:
-        - manage_index_templates
-      indices:
-        - names: 'logstash-*'
-          privileges:
-            - write
-            - delete
-            - create_index
-```                
-                
-* ```es_xpack_license``` - X-Pack license. The license is a json blob. Set the variable directly (possibly protected by Ansible vault) or from a file in the Ansible project on the control machine via a lookup:
-
-```
-es_xpack_license: "{{ lookup('file', playbook_dir + '/files/' + es_cluster_name + '/license.json') }}"
-``` 
-
-X-Pack configuration parameters can be added to the elasticsearch.yml file using the normal `es_config` parameter.
-
-For a full example see [here](https://github.com/elastic/ansible-elasticsearch/blob/master/test/integration/xpack.yml)
-
-#### Important Note for Native Realm Configuration
-
-In order for native users and roles to be configured, the role calls the Elasticsearch API.  Given security is installed this requires definition of two parameters:
-
-* ```es_api_basic_auth_username``` - admin username
-* ```es_api_basic_auth_password``` - admin password
-
-These can either be set to a user declared in the file based realm, with admin permissions, or the default "elastic" superuser (default password is changeme).
-
-
-### Additional Configuration
-
-In addition to es_config, the following parameters allow the customization of the Java and Elasticsearch versions as well as the role behaviour. Options include:
-
-* ```es_major_version```  Should be consistent with es_version. For versions >= 5.0 and < 6.0 this must be "5.x". For versions >= 6.0 this must be "6.x".
-* ```es_version``` (e.g. "6.1.2").
-* ```es_api_host``` The host name used for actions requiring HTTP e.g. installing templates. Defaults to "localhost".
-* ```es_api_port``` The port used for actions requiring HTTP e.g. installing templates. Defaults to 9200. **CHANGE IF THE HTTP PORT IS NOT 9200**
-* ```es_api_basic_auth_username``` The Elasticsearch username for making admin changing actions. Used if Security is enabled. Ensure this user is admin.
-* ```es_api_basic_auth_password``` The password associated with the user declared in `es_api_basic_auth_username`
-* ```es_start_service``` (true (default) or false)
-* ```es_plugins_reinstall``` (true or false (default) )
-* ```es_plugins``` an array of plugin definitions e.g.:
-```yml
-  es_plugins:
-    - plugin: ingest-geoip 
-```
-* ```es_allow_downgrades``` For development purposes only. (true or false (default) )
-* ```es_java_install``` If set to false, Java will not be installed. (true (default) or false)
-* ```update_java``` Updates Java to the latest version. (true or false (default))
-* ```es_max_map_count``` maximum number of VMA (Virtual Memory Areas) a process can own. Defaults to 262144.
-* ```es_max_open_files``` the maximum file descriptor number that can be opened by this process. Defaults to 65536.
-* ```es_max_threads``` the maximum number of threads the process can start. Defaults to 2048 (the minimum required by elasticsearch).
-* ```es_debian_startup_timeout``` how long Debian-family SysV init scripts wait for the service to start, in seconds. Defaults to 10 seconds.
-
-Earlier examples illustrate the installation of plugins using `es_plugins`.  For officially supported plugins no version or source delimiter is required. The plugin script will determine the appropriate plugin version based on the target Elasticsearch version.  For community based plugins include the full url.  This approach should NOT be used for the X-Pack plugin.  See X-Pack below for details here.
- 
-If installing Monitoring or Alerting, ensure the license plugin is also specified.  Security configuration is currently not supported but planned for later versions.
-
-* ```es_user``` - defaults to elasticsearch.
-* ```es_group``` - defaults to elasticsearch.
-* ```es_user_id``` - default is undefined.
-* ```es_group_id``` - default is undefined.
-
-Both ```es_user_id``` and ```es_group_id``` must be set for the user and group ids to be set. 
-
-By default, each node on a host will be installed to use unique pid, plugin, work, data and log directories.  These directories are created, using the instance and host name, beneath default locations ]
-controlled by the following parameters:
-
-* ```es_pid_dir``` - defaults to "/var/run/elasticsearch".
-* ```es_data_dirs``` - defaults to "/var/lib/elasticsearch".  This can be a list or comma separated string e.g. ["/opt/elasticsearch/data-1","/opt/elasticsearch/data-2"] or "/opt/elasticsearch/data-1,/opt/elasticsearch/data-2"
-* ```es_log_dir``` - defaults to "/var/log/elasticsearch".
-* ```es_restart_on_change``` - defaults to true.  If false, changes will not result in Elasticsearch being restarted.
-* ```es_plugins_reinstall``` - defaults to false.  If true, all currently installed plugins will be removed from a node.  Listed plugins will then be re-installed.  
-
-This role ships with sample scripts and templates located in the [files/scripts/](files/scripts) and [files/templates/](files/templates) directories, respectively. These variables are used with the Ansible [with_fileglob](http://docs.ansible.com/ansible/playbooks_loops.html#id4) loop. When setting the globs, be sure to use an absolute path.
-* ```es_scripts_fileglob``` - defaults to `<role>/files/scripts/`.
-* ```es_templates_fileglob``` - defaults to `<role>/files/templates/`.
-
-### Proxy
-
-To define proxy globaly, set the following variables:
-
-* ```es_proxy_host``` - global proxy host
-* ```es_proxy_port``` - global proxy port
-
-To define proxy only for a particular plugin during its installation:
-
-```
-  es_plugins:
-    - plugin: ingest-geoip 
-      proxy_host: proxy.example.com
-      proxy_port: 8080
-```
-
-> For plugins installation, proxy_host and proxy_port are used first if they are defined and fallback to the global proxy settings if not. The same values are currently used for both the http and https proxy settings.
-
-## Notes
-
-* The role assumes the user/group exists on the server.  The elasticsearch packages create the default elasticsearch user.  If this needs to be changed, ensure the user exists.
-* The playbook relies on the inventory_name of each host to ensure its directories are unique
-* Changing an instance_name for a role application will result in the installation of a new component.  The previous component will remain.
-* KitchenCI has been used for testing.  This is used to confirm images reach the correct state after a play is first applied.  We currently test only the latest version of 6.x on
-all supported platforms. 
-* The role aims to be idempotent.  Running the role multiple times, with no changes, should result in no state change on the server.  If the configuration is changed, these will be applied and 
-Elasticsearch restarted where required.
-* Systemd is used for Ubuntu versions >= 15, Debian >=8, Centos >=7.  All other versions use init for service scripts.
-* In order to run x-pack tests a license file with security enabled is required. A trial license is appropriate. Set the environment variable `ES_XPACK_LICENSE_FILE` to the full path of the license file prior to running tests.
-
-## IMPORTANT NOTES RE PLUGIN MANAGEMENT
-
-* If the ES version is changed, all plugins will be removed.  Those listed in the playbook will be re-installed.  This is behaviour is required in ES 6.x.
-* If no plugins are listed in the playbook for a node, all currently installed plugins will be removed.
-* The role supports automatic detection of differences between installed and listed plugins - installing those listed but not installed, and removing those installed but not listed.   Should users wish to re-install plugins they should set es_plugins_reinstall to true.  This will cause all currently installed plugins to be removed and those listed to be installed.
-
-## Questions on Usage
-
-We welcome questions on how to use the role.  However, in order to keep the github issues list focused on "issues" we ask the community to raise questions at https://discuss.elastic.co/c/elasticsearch.  This is monitored by the maintainers.
diff --git a/ansible/roles/es6/ansible.cfg b/ansible/roles/es6/ansible.cfg
deleted file mode 100644
index d9a8c50195..0000000000
--- a/ansible/roles/es6/ansible.cfg
+++ /dev/null
@@ -1 +0,0 @@
-[defaults]
\ No newline at end of file
diff --git a/ansible/roles/es6/defaults/main.yml b/ansible/roles/es6/defaults/main.yml
deleted file mode 100644
index f0ce74ac3d..0000000000
--- a/ansible/roles/es6/defaults/main.yml
+++ /dev/null
@@ -1,43 +0,0 @@
----
-es_major_version: "6.x"
-es_version: "6.2.3"
-es_version_lock: false
-es_use_repository: true
-es_templates_fileglob: "files/templates/*.json"
-es_apt_key: "https://artifacts.elastic.co/GPG-KEY-elasticsearch"
-es_apt_url: "deb https://artifacts.elastic.co/packages/{{ es_major_version }}/apt stable main"
-es_apt_url_old: "deb http://packages.elastic.co/elasticsearch/{{ es_major_version }}/debian stable main"
-es_start_service: true
-es_java_install: true
-update_java: false
-es_restart_on_change: true
-es_plugins_reinstall: false
-es_scripts: false
-es_templates: true
-es_user: elasticsearch
-es_group: elasticsearch
-es_config_log4j2: log4j2.properties.j2
-#Need to provide default directories
-es_pid_dir: "/var/run/elasticsearch"
-es_data_dirs: "/var/lib/elasticsearch"
-es_log_dir: "/var/log/elasticsearch"
-es_max_open_files: 65536
-es_max_threads: "{{ 2048 if ( es_version | version_compare('6.0.0', '<')) else 8192 }}"
-es_max_map_count: 262144
-es_allow_downgrades: false
-es_enable_xpack: false
-es_xpack_features: ["alerting","monitoring","graph","ml"]
-#These are used for internal operations performed by ansible.
-#They do not affect the current configuration
-es_api_host: "localhost"
-es_api_port: 9200
-es_debian_startup_timeout: 10
-
-# Since ansible 2.2 the following variables need to be defined
-# to allow the role to be conditionally played with a when condition.
-pid_dir: ''
-log_dir: ''
-conf_dir: ''
-data_dirs: ''
-# JVM custom parameters
-es_jvm_custom_parameters: ''
diff --git a/ansible/roles/es6/files/logging/log4j2.properties.custom.j2 b/ansible/roles/es6/files/logging/log4j2.properties.custom.j2
deleted file mode 100644
index 9a2a60f885..0000000000
--- a/ansible/roles/es6/files/logging/log4j2.properties.custom.j2
+++ /dev/null
@@ -1,76 +0,0 @@
-#CUSTOM LOG4J FILE
-
-status = error
-
-# log action execution errors for easier debugging
-logger.action.name = org.elasticsearch.action
-logger.action.level = info
-
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-appender.rolling.fileName = ${sys:es.logs}.log
-appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-
-rootLogger.level = debug
-rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling
-
-appender.deprecation_rolling.type = RollingFile
-appender.deprecation_rolling.name = deprecation_rolling
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
-appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
-appender.deprecation_rolling.policies.type = Policies
-appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.deprecation_rolling.policies.size.size = 10mb
-appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
-appender.deprecation_rolling.strategy.max = 4
-
-logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = debug
-logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
-logger.deprecation.additivity = false
-
-appender.index_search_slowlog_rolling.type = RollingFile
-appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
-appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
-appender.index_search_slowlog_rolling.policies.type = Policies
-appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_search_slowlog_rolling.policies.time.interval = 1
-appender.index_search_slowlog_rolling.policies.time.modulate = true
-
-logger.index_search_slowlog_rolling.name = index.search.slowlog
-logger.index_search_slowlog_rolling.level = debug
-logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
-logger.index_search_slowlog_rolling.additivity = false
-
-appender.index_indexing_slowlog_rolling.type = RollingFile
-appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
-appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-appender.index_indexing_slowlog_rolling.policies.type = Policies
-appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_indexing_slowlog_rolling.policies.time.interval = 1
-appender.index_indexing_slowlog_rolling.policies.time.modulate = true
-
-logger.index_indexing_slowlog.name = index.indexing.slowlog.index
-logger.index_indexing_slowlog.level = debug
-logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
-logger.index_indexing_slowlog.additivity = false
diff --git a/ansible/roles/es6/files/mappings/learning.json b/ansible/roles/es6/files/mappings/learning.json
deleted file mode 100644
index c5991a3d59..0000000000
--- a/ansible/roles/es6/files/mappings/learning.json
+++ /dev/null
@@ -1,783 +0,0 @@
-{
-  "index_patterns": "learning-*",
-  "settings": {
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "events_v1": {
-      "dynamic": false,
-      "properties": {
-        "eid": {
-          "type": "keyword"
-        },
-        "ets": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "mid": {
-          "type": "keyword"
-        },
-        "uid": {
-          "type": "keyword"
-        },
-        "ts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "syncts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "ver": {
-          "type": "keyword"
-        },
-        "context":{
-          "properties": {
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "model": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "granularity": {
-              "type": "keyword"
-            },
-            "date_range": {
-              "properties": {
-                "from": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "to": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                }
-              }
-            }
-          }
-        },
-        "dimensions": {
-          "properties": {
-            "did": {
-              "type": "keyword"
-            },
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "pid": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "gdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "sid": {
-              "type": "keyword"
-            },
-            "channel": {
-              "type": "keyword"
-            },
-            "group_user": {
-              "type": "boolean"
-            },
-            "anonymous_user": {
-              "type": "boolean"
-            },
-            "uid": {
-              "type": "keyword"
-            },
-            "ss_mid": {
-              "type": "keyword"
-            },
-            "tag": {
-              "type": "keyword"
-            },
-            "period": {
-              "type": "integer"
-            },
-            "content_id": {
-              "type": "keyword"
-            },
-            "item_id": {
-              "type": "keyword"
-            },
-            "author_id": {
-              "type": "keyword"
-            },
-            "loc": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "mode": {
-              "type": "keyword"
-            },
-            "context_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            },
-            "object_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            }
-          }
-        },
-        "edata": {
-          "properties": {
-            "eks": {
-              "properties": {
-                "total_users_count": {
-                  "type": "long"
-                },
-                "total_devices_count": {
-                  "type": "long"
-                },
-                "total_content_count": {
-                  "type": "long"
-                },
-                "content_ids": {
-                  "type": "keyword"
-                },
-                "device_ids": {
-                  "type": "keyword"
-                },
-                "user_ids": {
-                  "type": "keyword"
-                },
-                "contents": {
-                  "type": "keyword"
-                },
-                "unique_users": {
-                  "type": "keyword"
-                },
-                "avg_ts_session": {
-                  "type": "double"
-                },
-                "total_sessions": {
-                  "type": "long"
-                },
-                "avg_interactions_min": {
-                  "type": "double"
-                },
-                "total_interactions": {
-                  "type": "long"
-                },
-                "avg_pageviews": {
-                  "type": "double"
-                },
-                "total_ts": {
-                  "type": "double"
-                },
-                "time_diff": {
-                  "type": "double"
-                },
-                "time_spent": {
-                  "type": "double"
-                },
-                "interact_events_per_min": {
-                  "type": "double"
-                },
-                "interact_events_count": {
-                  "type": "long"
-                },
-                "total_pageviews_count": {
-                  "type": "long"
-                },
-                "page_views_count": {
-                  "type": "long"
-                },
-                "ce_visits": {
-                  "type": "long"
-                },
-                "start_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "end_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "syncDate": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "timeSpent": {
-                  "type": "double"
-                },
-                "interactEventsPerMin": {
-                  "type": "double"
-                },
-                "mimeType": {
-                  "type": "keyword"
-                },
-                "contentType": {
-                  "type": "keyword"
-                },
-                "timeDiff": {
-                  "type": "double"
-                },
-                "mode": {
-                  "type": "keyword"
-                },
-                "telemetryVersion": {
-                  "type": "keyword"
-                },
-                "noOfInteractEvents": {
-                  "type": "long"
-                },
-                "interruptTime": {
-                  "type": "double"
-                },
-                "contentCount": {
-                  "type": "long"
-                },
-                "content": {
-                  "type": "keyword"
-                },
-                "time_stamp": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "first_visit": {
-                  "type": "boolean"
-                },
-                "noOfAttempts": {
-                  "type": "integer"
-                },
-                "m_side_loads": {
-                  "type": "integer"
-                },
-                "m_downloads": {
-                  "type": "integer"
-                },
-                "m_avg_rating": {
-                  "type": "double"
-                },
-                "m_ratings": {
-                  "type": "nested",
-                  "properties": {
-                    "rating": {
-                      "type": "integer"
-                    },
-                    "time": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    }
-                  }
-                },
-                "m_comments": {
-                  "type": "object"
-                },
-                "ce_total_ts": {
-                  "type": "double"
-                },
-                "ce_percent_sessions": {
-                  "type": "double"
-                },
-                "ce_visits_count": {
-                  "type": "long"
-                },
-                "ce_total_visits": {
-                  "type": "long"
-                },
-                "ce_percent_ts": {
-                  "type": "double"
-                },
-                "avg_ts": {
-                  "type": "double"
-                },
-                "inc_res_count": {
-                  "type": "long"
-                },
-                "incorrect_res": {
-                  "type": "nested",
-                  "properties": {
-                    "resp": {
-                      "type": "keyword"
-                    },
-                    "mmc": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "total_count": {
-                  "type": "long"
-                },
-                "correct_res_count": {
-                  "type": "long"
-                },
-                "correct_res": {
-                  "type": "keyword"
-                },
-                "itemId": {
-                  "type": "keyword"
-                },
-                "exTimeSpent": {
-                  "type": "double"
-                },
-                "score": {
-                  "type": "integer"
-                },
-                "pass": {
-                  "type": "keyword"
-                },
-                "qtitle": {
-                  "type": "keyword"
-                },
-                "qdesc": {
-                  "type": "keyword"
-                },
-                "new_user_count": {
-                  "type": "long"
-                },
-                "anon_total_sessions": {
-                  "type": "long"
-                },
-                "anon_total_ts": {
-                  "type": "double"
-                },
-                "anon_avg_ts_session": {
-                  "type": "double"
-                },
-                "percent_new_users_count": {
-                  "type": "double"
-                },
-                "ce_total_sessions": {
-                  "type": "long"
-                },
-                "unique_users_count": {
-                  "type": "long"
-                },
-                "stageVisitCount": {
-                  "type": "long"
-                },
-                "interactEventsCount": {
-                  "type": "long"
-                },
-                "interactEvents": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    }
-                  }
-                },
-                "stageId": {
-                  "type": "keyword"
-                },
-                "visitCount": {
-                  "type": "long"
-                },
-                "num_downloads": {
-                  "type": "long"
-                },
-                "num_sideloads": {
-                  "type": "long"
-                },
-                "avg_depth": {
-                  "type": "double"
-                },
-                "publish_pipeline_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "state": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "events_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "eventsSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "screenSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "visitCount": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "activitySummary": {
-                  "type": "nested",
-                  "properties": {
-                    "actType": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "page_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "visit_count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "env_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "itemResponses": {
-                  "type": "nested",
-                  "properties": {
-                    "itemId": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "exTimeSpent": {
-                      "type": "double"
-                    },
-                    "score": {
-                      "type": "integer"
-                    },
-                    "time_stamp": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    },
-                    "pass": {
-                      "type": "keyword"
-                    },
-                    "qtitle": {
-                      "type": "keyword"
-                    },
-                    "qdesc": {
-                      "type": "keyword"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "etags": {
-          "properties": {
-            "app": {
-              "type": "keyword"
-            },
-            "partner": {
-              "type": "keyword"
-            },
-            "dims": {
-              "type": "keyword"
-            }
-          }
-        },
-        "geoip": {
-          "properties": {
-            "location": {
-              "type": "geo_point"
-            }
-          },
-          "type": "object"
-        },
-        "contentdata": {
-          "properties": {
-            "ageGroup": {
-              "type": "keyword"
-            },
-            "author": {
-              "type": "keyword"
-            },
-            "audience": {
-              "type": "keyword"
-            },
-            "code": {
-              "type": "keyword"
-            },
-            "collaborators": {
-              "type": "keyword"
-            },
-            "collections": {
-              "type": "keyword"
-            },
-            "concepts": {
-              "type": "keyword"
-            },
-            "contentType": {
-              "type": "keyword"
-            },
-            "curriculum": {
-              "type": "keyword"
-            },
-            "developer": {
-              "type": "keyword"
-            },
-            "domain": {
-              "type": "keyword"
-            },
-            "downloadUrl": {
-              "type": "keyword"
-            },
-            "downloads": {
-              "type": "long"
-            },
-            "edition": {
-              "type": "keyword"
-            },
-            "genre": {
-              "type": "keyword"
-            },
-            "gradeLevel": {
-              "type": "keyword"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "me_totalDevices": {
-              "type": "long"
-            },
-            "me_totalDownloads": {
-              "type": "long"
-            },
-            "me_totalInteractions": {
-              "type": "long"
-            },
-            "me_totalRatings": {
-              "type": "long"
-            },
-            "me_totalSessionsCount": {
-              "type": "long"
-            },
-            "me_totalSideloads": {
-              "type": "long"
-            },
-            "me_totalTimespent": {
-              "type": "long"
-            },
-            "me_totalUsage": {
-              "type": "long"
-            },
-            "medium": {
-              "type": "keyword"
-            },
-            "methods": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "popularity": {
-              "type": "long"
-            },
-            "portalOwner": {
-              "type": "keyword"
-            },
-            "publication": {
-              "type": "keyword"
-            },
-            "publisher": {
-              "type": "keyword"
-            },
-            "rating": {
-              "type": "long"
-            },
-            "size": {
-              "type": "long"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "subject": {
-              "type": "keyword"
-            },
-            "templateType": {
-              "type": "keyword"
-            },
-            "theme": {
-              "type": "keyword"
-            },
-            "words": {
-              "type": "keyword"
-            }
-          }
-        },
-        "itemdata": {
-          "properties": {
-            "concepts": {
-              "type": "keyword"
-            },
-            "createdBy": {
-              "type": "keyword"
-            },
-            "createdOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "language": {
-              "type": "keyword"
-            },
-            "lastUpdatedBy": {
-              "type": "keyword"
-            },
-            "lastUpdatedOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "media": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "num_answers": {
-              "type": "long"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "qlevel": {
-              "type": "keyword"
-            },
-            "question": {
-              "type": "keyword"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "template": {
-              "type": "keyword"
-            },
-            "title": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "long"
-            }
-          }
-        },
-        "@version": {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/scripts/calculate-score.groovy b/ansible/roles/es6/files/scripts/calculate-score.groovy
deleted file mode 100644
index 442c25ca6c..0000000000
--- a/ansible/roles/es6/files/scripts/calculate-score.groovy
+++ /dev/null
@@ -1 +0,0 @@
-log(_score * 2) + my_modifier
\ No newline at end of file
diff --git a/ansible/roles/es6/files/system_key b/ansible/roles/es6/files/system_key
deleted file mode 100644
index 91962910d2..0000000000
Binary files a/ansible/roles/es6/files/system_key and /dev/null differ
diff --git a/ansible/roles/es6/files/templates/backend.json b/ansible/roles/es6/files/templates/backend.json
deleted file mode 100644
index 07161ca348..0000000000
--- a/ansible/roles/es6/files/templates/backend.json
+++ /dev/null
@@ -1,411 +0,0 @@
-{
-  "index_patterns" : "backend-*",
-  "settings" : {
-    "number_of_shards" : 5
-  },
-  "mappings" : {
-    "events" : {
-      "dynamic": false,
-      "properties": {
-          "@timestamp": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "@version": {
-            "type": "keyword"
-          },
-          "eid": {
-            "type": "keyword"
-          },
-          "ets": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "mid": {
-            "type": "keyword"
-          },
-          "ts": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "ver": {
-            "type": "keyword"
-          },
-          "actor": {
-             "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "type": {
-                  "type": "keyword"
-                }
-             }
-          },
-          "edata": {
-            "properties": {
-              "comments": {
-                "type": "keyword"
-              },
-              "correlationid": {
-                "type": "keyword"
-              },
-              "duration": {
-                "type": "double"
-              },
-              "data": {
-                "type": "keyword"
-              },
-              "dir": {
-                "type": "keyword"
-              },
-              "errtype": {
-                "type": "keyword"
-              },
-              "id": {
-                "type": "keyword"
-              },
-              "level": {
-                "type": "keyword"
-              },
-              "loc": {
-                "type": "keyword"
-              },
-              "message": {
-                "type": "keyword"
-              },
-              "mode": {
-                "type": "keyword"
-              },
-              "pass": {
-                "type": "keyword"
-              },
-              "prevstate": {
-                "type": "keyword"
-              },
-              "pageid": {
-                "type": "keyword"
-              },
-              "query": {
-                "type": "keyword"
-              },
-              "rating": {
-                "type": "double"
-              },
-              "score": {
-                "type": "double"
-              },
-              "size": {
-                "type": "double"
-              },
-              "state": {
-                "type": "keyword"
-              },
-              "subtype": {
-                "type": "keyword"
-              },
-              "type": {
-                "type": "keyword"
-              },
-              "uri": {
-                "type": "keyword"
-              },
-              "items": {
-                "type": "nested",
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "origin": {
-                    "properties": {
-                      "id": {
-                        "type": "keyword"
-                      },
-                      "type": {
-                        "type": "keyword"
-                      }
-                    }
-                  }
-                }
-              },
-              "item": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "maxscore": {
-                    "type": "long"
-                  },
-                  "exlength": {
-                    "type": "long"
-                  },
-                  "uri": {
-                    "type": "keyword"
-                  },
-                  "desc": {
-                    "type": "keyword"
-                  },
-                  "title": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "target": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "category": {
-                    "type": "keyword"
-                  },
-                  "parent": {
-                    "properties": {
-                      "id": {
-                        "type": "keyword"
-                      },
-                      "type": {
-                        "type": "keyword"
-                      }
-                    }
-                  }
-                }
-              },
-              "visits": {
-                "type": "nested",
-                "properties": {
-                  "objid": {
-                    "type": "keyword"
-                  },
-                  "objtype": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "plugin": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "category": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "object": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "subtype": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "uaspec": {
-                "properties": {
-                  "agent": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "system": {
-                    "type": "keyword"
-                  },
-                  "platform": {
-                    "type": "keyword"
-                  },
-                  "raw": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "dspec": {
-                "properties": {
-                   "camera": {
-                      "type": "keyword"
-                    },
-                    "cpu": {
-                      "type": "keyword"
-                    },
-                    "edisk": {
-                      "type": "double"
-                    },
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "idisk": {
-                      "type": "double"
-                    },
-                    "make": {
-                      "type": "keyword"
-                    },
-                    "mem": {
-                      "type": "double"
-                    },
-                    "os": {
-                      "type": "keyword"
-                    },
-                    "scrn": {
-                      "type": "double"
-                    },
-                    "sims": {
-                      "type": "double"
-                    }
-                }
-              }
-            }
-          },
-          "context": {
-            "properties": {
-              "channel": {
-                "type": "keyword"
-              },
-              "env": {
-                "type": "keyword"
-              },
-              "sid": {
-                "type": "keyword"
-              },
-              "did": {
-                "type": "keyword"
-              },
-              "pdata": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "pid": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "cdata": {
-                "type": "nested",
-                "properties": {
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "id": {
-                    "type": "keyword"
-                  }
-                }
-              }
-            }
-          },
-          "object": {
-            "properties": {
-              "id": {
-                "type": "keyword"
-              },
-              "type": {
-                "type": "keyword"
-              },
-              "subtype": {
-                "type": "keyword"
-              },
-              "parentid": {
-                "type": "keyword"
-              },
-              "parenttype": {
-                "type": "keyword"
-              },
-              "ver": {
-                "type": "keyword"
-              },
-              "rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-              }
-            }
-          },
-          "metadata": {
-            "properties": {
-              "source": {
-                "type": "keyword"
-              },
-              "index_name": {
-                "type": "keyword"
-              },
-              "index_type": {
-                "type": "keyword"
-              },
-              "source_eid": {
-                "type": "keyword"
-              },
-              "source_mid": {
-                "type": "keyword"
-              },
-              "pump": {
-                "type": "keyword"
-              }
-            }
-          },
-          "flags": {
-            "properties": {
-              "v2_converted": {
-                "type": "boolean"
-              },
-              "dd_processed": {
-                "type": "boolean"
-              },
-              "tv_processed": {
-                "type": "boolean"
-              }
-            }
-          }
-       }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/templates/compositeindex.json b/ansible/roles/es6/files/templates/compositeindex.json
deleted file mode 100644
index d19a189948..0000000000
--- a/ansible/roles/es6/files/templates/compositeindex.json
+++ /dev/null
@@ -1,5122 +0,0 @@
-{
-  "index_patterns": "compositesearch",
-  "settings": {
-    "index": {
-      "max_ngram_diff" : "29",
-      "analysis": {
-        "filter": {
-          "mynGram": {
-            "token_chars": [
-              "letter",
-              "digit",
-              "whitespace",
-              "punctuation",
-              "symbol"
-            ],
-            "min_gram": "1",
-            "type": "nGram",
-            "max_gram": "30"
-          }
-        },
-        "analyzer": {
-          "cs_index_analyzer": {
-            "filter": [
-              "lowercase",
-              "mynGram"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "keylower": {
-            "filter": "lowercase",
-            "tokenizer": "keyword"
-          },
-          "cs_search_analyzer": {
-            "filter": [
-              "standard",
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          }
-        }
-      }
-    },
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "cs": {
-      "dynamic_templates": [
-        {
-          "longs": {
-            "mapping": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "match_mapping_type": "long"
-          }
-        },
-        {
-          "booleans": {
-            "mapping": {
-              "type": "boolean",
-              "fields": {
-                "raw": {
-                  "type": "boolean"
-                }
-              }
-            },
-            "match_mapping_type": "boolean"
-          }
-        },
-        {
-          "doubles": {
-            "mapping": {
-              "type": "double",
-              "fields": {
-                "raw": {
-                  "type": "double"
-                }
-              }
-            },
-            "match_mapping_type": "double"
-          }
-        },
-        {
-          "dates": {
-            "mapping": {
-              "type": "date",
-              "fields": {
-                "raw": {
-                  "type": "date"
-                }
-              }
-            },
-            "match_mapping_type": "date"
-          }
-        },
-        {
-          "strings": {
-            "mapping": {
-              "type": "text",
-              "copy_to": "all_fields",
-              "analyzer": "cs_index_analyzer",
-              "search_analyzer": "cs_search_analyzer",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              }
-            },
-            "match_mapping_type": "string"
-          }
-        },
-        {
-          "nested": {
-            "mapping": {
-              "type": "nested",
-              "fields": {
-                "type": "nested"
-              }
-            },
-            "match_mapping_type": "object"
-          }
-        }
-      ],
-      "properties": {
-        "Audio": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_FUNC_OBJECT_TYPE": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_INDEXABLE_METADATA_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_IN_RELATIONS_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_NON_INDEXABLE_METADATA_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_OUT_RELATIONS_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_REQUIRED_PROPERTIES": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_SYSTEM_TAGS_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_SYS_NODE_TYPE": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "IL_UNIQUE_ID": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "Link": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "RhymingSound": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "SET_OBJECT_TYPE_KEY": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "SET_TYPE": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "SYS_INTERNAL_LAST_UPDATED_ON": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "TemplateType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "actions": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "activityType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "activity_class": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ageGroup": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "all_fields": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "allowupdate_status": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "altIsoSymbol": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "answer": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "antonyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "apiId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "appIcon": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "appIconLabel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "appId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "applicability": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "artifactUrl": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "attribution": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "attributions": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "audience": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "author": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "authoringScore": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "averageComplexity": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "avgGamingTime": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "badgeAssertions": {
-          "properties": {
-            "assertionId": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "cs_index_analyzer",
-              "search_analyzer": "cs_search_analyzer"
-            },
-            "badgeClassId": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "cs_index_analyzer",
-              "search_analyzer": "cs_search_analyzer"
-            },
-            "badgeClassName": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "cs_index_analyzer",
-              "search_analyzer": "cs_search_analyzer"
-            },
-            "createdTS": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "issuerId": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "cs_index_analyzer",
-              "search_analyzer": "cs_search_analyzer"
-            }
-          }
-        },
-        "baseUrl": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "bloomsTaxonomyLevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "board": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "body": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "c_null_open_batch_count": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "c_null_private_batch_count": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "c_test_name_open_batch_count": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "cases": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "categories": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "category": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "categoryinstances": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "channel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "channels": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "chapterName": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "chapterNumber": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "childNodes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "children": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "class": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "code": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "collaborators": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "collections": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "communication_scheme": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "compatibilityLevel": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "conceptData": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "conceptIds": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "concepts": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "consumerGroups": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "consumerId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentDisposition": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentEncoding": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentFilter": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentMeta": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contentTypesCount": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "contents": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "converse": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "copyType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "copyright": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "cost": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "count_grade_1": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_grade_2": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_grade_3": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_CC": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_DEM": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_ECH": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_INJ": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_INTF": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_JJ": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_NEG": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_NN": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_NST": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_PRP": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_PSP": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_QC": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_QF": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_QO": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_RB": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_RDP": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_RP": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_VAUX": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_VM": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_pos_WQ": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_source_Karnataka_Govt_Textbooks": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_source_Rajasthan_Textbooks": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "count_textbooks": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "createdBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "createdFor": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "createdOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "creator": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "curriculum": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "defaultFramework": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "defaultRes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "description": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "developer": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "dialcodes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "dimensions": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "domain": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "downloadUrl": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "duration": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "edition": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "editorState": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ekstepWordnet": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "endPoint": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "endsWithAkshara": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "example": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "exampleSentences": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "externalValidation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "faculty": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "failPopup": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "feedback": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "fields": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "filter": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "flagReasons": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "flaggedBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "flags": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "follows": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "format": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "foundation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "framework": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "frameworkId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "frameworks": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "fw_hierarchy": {
-          "type": "text",
-          "index": false
-        },
-        "gender": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "genders": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "genieScore": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "genre": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "gloss": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "grade": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "gradeLevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "graph_id": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "grayScaleAppIcon": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "hasAntonyms": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "hasConnotative": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "hasSynonyms": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "hints": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "holonyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "hypernyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "hyponyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "i18n": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "idealScreenDensity": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "idealScreenSize": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "identifier": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "illustrator": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "illustrators": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "image": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "imageCredits": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "index": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "indowordnetId": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "inflections": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "instruction": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "instructions": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "interactivityLevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "internalValidation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ipaSymbol": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "isConnotative": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "isLoanWord": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "isPhrase": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "isoCode": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "isoSymbol": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "itemType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "item_sets": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "items": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "keywords": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "keywords_bkp": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "label": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "langid": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "language": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "languageCode": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "languageId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "languageLevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "lastFlaggedOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "lastPublishDate": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "lastPublishedBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "lastPublishedOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "lastSubmittedOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "lastUpdatedBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "lastUpdatedOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "launchUrl": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "leafNodesCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "learnerLevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "learningObjective": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "lemma": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "level": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "lhs_options": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "license": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "limit": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "liveWords": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "material": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "maxChainLength": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "maxWords": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "max_score": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_audiosCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_averageInteractionsPerMin": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_averageRating": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_averageSessionsPerDevice": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_averageTimespentPerSession": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_avgCreationTsPerSession": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_creationSessions": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_creationTimespent": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_imagesCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_timespentDraft": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_timespentReview": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_totalComments": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalDevices": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalDownloads": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalInteractions": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalRatings": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalSessionsCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalSideloads": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "me_totalTimespent": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "me_videosCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "meaning": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "media": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "mediaType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "mediatype": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "medium": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "meronyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "method": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "methods": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "mimeType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "mimeTypesCount": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "minChainLength": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "model": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "model_sample": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "morphology": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "name": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "nodeType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "node_id": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "notes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "num_answers": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "objectType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "objects": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "objectsUsed": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "occurrenceCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "oldContentBody": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "optStatus": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "options": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "organization": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "origin": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "orthographic_complexity": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "os": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "osId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "owner": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pageNumber": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "parent": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "partial_scoring": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "person": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "phonologic_complexity": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "pictures": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pkgVersion": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "plurality": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "popularity": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "portalOwner": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pos": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "posTags": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pos_categories": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "posterImage": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pragma": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "prevState": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "previewUrl": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "primaryMeaning": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "primaryMeaningId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "product": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "pronunciations": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "publication": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "publishError": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "publish_type": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "publisher": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "purpose": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "qid": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "qlevel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "qtype": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "question": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "questionImage": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "question_audio": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "question_count": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "question_image": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "references": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "renderingHints": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "resourceType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "resources": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "responses": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "rhs_options": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "rownum": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "ruleId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ruleObjectType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ruleScript": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "s3Key": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "sYS_INTERNAL_LAST_UPDATED_ON": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "sampleUsages": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "savingPopup": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "screenshots": {
-          "type": "text",
-          "index": false
-        },
-        "semanticVersion": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "showNotification": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "shuffle": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "size": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "skills": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "softConstraints": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "soundCredits": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "source": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "sourceTypes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "sources": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "startWordsSize": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "startsWithAkshara": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "state": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "status": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "strict_sequencing": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "subject": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "sublevel": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "successPopup": {
-          "type": "boolean",
-          "fields": {
-            "raw": {
-              "type": "boolean"
-            }
-          }
-        },
-        "syllableCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "syllableNotation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "syllables": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "synonyms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "synsetCount": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "synsets": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "tags": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "tempData": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "template": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "templateName": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "templateType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "template_id": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "terms": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "text": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "textbookName": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "theme": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "themes": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "thumbnail": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "title": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "toc_url": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "tools": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "totalGamingTime": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "total_items": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "translations": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "ttl": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "tutor": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "type": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "unicode": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "unicodeNotation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "usedByContent": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "used_for": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "usesContent": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "variants": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "varna": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "version": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "versionCheckMode": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "versionKey": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "visibility": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "voiceCredits": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "vowel": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "weightages": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "wordImages": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "wordListIds": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "wordLists": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "word_complexity": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "words": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "wordsets": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "workers": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        },
-        "year": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "cs_index_analyzer",
-          "search_analyzer": "cs_search_analyzer"
-        }
-      }
-    }
-  }
-}
diff --git a/ansible/roles/es6/files/templates/dialcode.json b/ansible/roles/es6/files/templates/dialcode.json
deleted file mode 100644
index 12e600ad94..0000000000
--- a/ansible/roles/es6/files/templates/dialcode.json
+++ /dev/null
@@ -1,269 +0,0 @@
-{
-  "index_patterns": "dialcode",
-  "settings": {
-    "index": {
-      "analysis": {
-        "filter": {
-          "mynGram": {
-            "token_chars": [
-              "letter",
-              "digit",
-              "whitespace",
-              "punctuation",
-              "symbol"
-            ],
-            "min_gram": "1",
-            "type": "nGram",
-            "max_gram": "20"
-          }
-        },
-        "analyzer": {
-          "dc_search_analyzer": {
-            "filter": [
-              "standard",
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "dc_index_analyzer": {
-            "filter": [
-              "lowercase",
-              "mynGram"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "keylower": {
-            "filter": "lowercase",
-            "tokenizer": "keyword"
-          }
-        }
-      }
-    },
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "dc": {
-      "dynamic_templates": [
-        {
-          "longs": {
-            "mapping": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "match_mapping_type": "long"
-          }
-        },
-        {
-          "booleans": {
-            "mapping": {
-              "type": "boolean",
-              "fields": {
-                "raw": {
-                  "type": "boolean"
-                }
-              }
-            },
-            "match_mapping_type": "boolean"
-          }
-        },
-        {
-          "doubles": {
-            "mapping": {
-              "type": "double",
-              "fields": {
-                "raw": {
-                  "type": "double"
-                }
-              }
-            },
-            "match_mapping_type": "double"
-          }
-        },
-        {
-          "dates": {
-            "mapping": {
-              "type": "date",
-              "fields": {
-                "raw": {
-                  "type": "date"
-                }
-              }
-            },
-            "match_mapping_type": "date"
-          }
-        },
-        {
-          "strings": {
-            "mapping": {
-              "type": "text",
-              "copy_to": "all_fields",
-              "analyzer": "dc_index_analyzer",
-              "search_analyzer": "dc_search_analyzer",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              }
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "properties": {
-        "all_fields": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "batchCode": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "batchcode": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "channel": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "dialcode_index": {
-          "type": "double",
-          "fields": {
-            "raw": {
-              "type": "double"
-            }
-          }
-        },
-        "generated_on": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "identifier": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "objectType": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "published_on": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "publisher": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        },
-        "status": {
-          "type": "text",
-          "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "dc_index_analyzer",
-          "search_analyzer": "dc_search_analyzer"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/templates/experimentindex.json b/ansible/roles/es6/files/templates/experimentindex.json
deleted file mode 100644
index cda5da556a..0000000000
--- a/ansible/roles/es6/files/templates/experimentindex.json
+++ /dev/null
@@ -1,185 +0,0 @@
-{
-  "index_patterns": "experiment",
-  "settings": {
-    "index": {
-      "number_of_shards": "5",
-      "analysis": {
-        "analyzer": {
-          "exp_index_analyzer": {
-            "filter": [
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "keylower": {
-            "filter": "lowercase",
-            "type": "custom",
-            "tokenizer": "keyword"
-          },
-          "exp_search_analyzer": {
-            "filter": [
-              "lowercase",
-              "standard"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          }
-        }
-      },
-      "number_of_replicas": "1"
-    }
-  },
-  "mappings" : {
-    "exp" : {
-      "dynamic" : "false",
-      "properties" : {
-        "all_fields" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower"
-            }
-          },
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer"
-        },
-        "id" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower",
-              "fielddata" : true
-            }
-          },
-          "copy_to" : [
-            "all_fields"
-          ],
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer",
-          "fielddata" : true
-        },
-        "ver" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower",
-              "fielddata" : true
-            }
-          },
-          "copy_to" : [
-            "all_fields"
-          ],
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer",
-          "fielddata" : true
-        },
-        "expType" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower",
-              "fielddata" : true
-            }
-          },
-          "copy_to" : [
-            "all_fields"
-          ],
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer",
-          "fielddata" : true
-        },
-        "name" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower",
-              "fielddata" : true
-            }
-          },
-          "copy_to" : [
-            "all_fields"
-          ],
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer",
-          "fielddata" : true
-        },
-        "userId" : {
-          "type" : "keyword"
-        },
-        "userIdMod" : {
-          "type" : "long"
-        },
-        "extData" : {
-          "type" : "text",
-          "fields" : {
-            "raw" : {
-              "type" : "text",
-              "analyzer" : "keylower",
-              "fielddata" : true
-            }
-          },
-          "copy_to" : [
-            "all_fields"
-          ],
-          "analyzer" : "exp_index_analyzer",
-          "search_analyzer" : "exp_search_analyzer",
-          "fielddata" : true
-        },
-        "createdOn" : {
-          "type" : "date",
-          "fields" : {
-            "raw" : {
-              "type" : "date",
-              "format": "date_hour_minute_second"
-            }
-          }
-        },
-        "startDate" : {
-          "type" : "date",
-          "fields" : {
-            "raw" : {
-              "type" : "date",
-              "format": "date_hour_minute_second"
-            }
-          }
-        },
-        "endDate" : {
-          "type" : "date",
-          "fields" : {
-            "raw" : {
-              "type" : "date",
-              "format": "date_hour_minute_second"
-            }
-          }
-        },
-        "lastUpdatedOn" : {
-          "type" : "date",
-          "fields" : {
-            "raw" : {
-              "type" : "date",
-              "format": "date_hour_minute_second"
-            }
-          }
-        },
-        "deviceId" : {
-          "type" : "keyword"
-        },
-        "deviceIdMod" : {
-          "type" : "long"
-        },
-        "key" : {
-          "type" : "keyword"
-        },
-        "url" : {
-          "type" : "keyword"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/templates/failed-telemetry.json b/ansible/roles/es6/files/templates/failed-telemetry.json
deleted file mode 100644
index 5d474e7231..0000000000
--- a/ansible/roles/es6/files/templates/failed-telemetry.json
+++ /dev/null
@@ -1,106 +0,0 @@
-{
-  "index_patterns" : "failed-telemetry-*",
-  "settings" : {
-    "number_of_shards" : 5
-  },
-  "mappings" : {
-    "events" : {
-      "dynamic": false,
-       "properties": {
-          "@timestamp": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "@version": {
-            "type": "keyword"
-          },
-          "channel": {
-            "type": "keyword"
-          },
-          "eid": {
-            "type": "keyword"
-          },
-          "ets": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "mid": {
-            "type": "keyword"
-          },
-          "ts": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "metadata": {
-            "properties": {
-              "source": {
-                "type": "keyword"
-              },
-              "index_name": {
-                "type": "keyword"
-              },
-              "index_type": {
-                "type": "keyword"
-              }
-            }
-          },
-          "context": {
-            "properties": {
-              "channel": {
-                "type": "keyword"
-              },
-              "env": {
-                "type": "keyword"
-              },
-              "sid": {
-                "type": "keyword"
-              },
-              "did": {
-                "type": "keyword"
-              },
-              "pdata": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "pid": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "cdata": {
-                "type": "nested",
-                "properties": {
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "id": {
-                    "type": "keyword"
-                  }
-                }
-              }
-            }
-          }
-        }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/templates/lp_audit_log.json b/ansible/roles/es6/files/templates/lp_audit_log.json
deleted file mode 100644
index b33cf788b0..0000000000
--- a/ansible/roles/es6/files/templates/lp_audit_log.json
+++ /dev/null
@@ -1,314 +0,0 @@
-{
-  "index_patterns": "lp_audit_log",
-  "settings": {
-    "index": {
-      "analysis": {
-        "filter": {
-          "mynGram": {
-            "token_chars": [
-              "letter",
-              "digit",
-              "whitespace",
-              "punctuation",
-              "symbol"
-            ],
-            "min_gram": "1",
-            "type": "nGram",
-            "max_gram": "20"
-          }
-        },
-        "analyzer": {
-          "ah_search_analyzer": {
-            "filter": [
-              "standard",
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "keylower": {
-            "filter": "lowercase",
-            "tokenizer": "keyword"
-          },
-          "ah_index_analyzer": {
-            "filter": [
-              "lowercase",
-              "mynGram"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          }
-        }
-      }
-    },
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "ah": {
-      "dynamic_templates": [
-        {
-          "longs": {
-            "mapping": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "match_mapping_type": "long"
-          }
-        },
-        {
-          "booleans": {
-            "mapping": {
-              "type": "boolean",
-              "fields": {
-                "raw": {
-                  "type": "boolean"
-                }
-              }
-            },
-            "match_mapping_type": "boolean"
-          }
-        },
-        {
-          "doubles": {
-            "mapping": {
-              "type": "double",
-              "fields": {
-                "raw": {
-                  "type": "double"
-                }
-              }
-            },
-            "match_mapping_type": "double"
-          }
-        },
-        {
-          "dates": {
-            "mapping": {
-              "type": "date",
-              "fields": {
-                "raw": {
-                  "type": "date"
-                }
-              }
-            },
-            "match_mapping_type": "date"
-          }
-        },
-        {
-          "strings": {
-            "mapping": {
-              "type": "text",
-              "copy_to": "all_fields",
-              "analyzer": "ah_index_analyzer",
-              "search_analyzer": "ah_search_analyzer",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              }
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "properties": {
-        "@timestamp": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "@version": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "all_fields": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "audit_id": {
-          "type": "long",
-          "fields": {
-            "raw": {
-              "type": "long"
-            }
-          }
-        },
-        "createdOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "graphId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "label": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "logRecord": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "objectId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "objectType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "operation": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "requestId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "summary": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        },
-        "userId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "ah_index_analyzer",
-          "search_analyzer": "ah_search_analyzer"
-        }
-      }
-    }
-  }
-}
diff --git a/ansible/roles/es6/files/templates/suggestionindex.json b/ansible/roles/es6/files/templates/suggestionindex.json
deleted file mode 100644
index cb32559b1b..0000000000
--- a/ansible/roles/es6/files/templates/suggestionindex.json
+++ /dev/null
@@ -1,784 +0,0 @@
-{
-  "index_patterns": "lp_suggestion_index_v1",
-  "settings": {
-    "index": {
-      "analysis": {
-        "filter": {
-          "mynGram": {
-            "token_chars": [
-              "letter",
-              "digit",
-              "whitespace",
-              "punctuation",
-              "symbol"
-            ],
-            "min_gram": "1",
-            "type": "nGram",
-            "max_gram": "20"
-          }
-        },
-        "analyzer": {
-          "keylower": {
-            "filter": "lowercase",
-            "tokenizer": "keyword"
-          },
-          "sg_index_analyzer": {
-            "filter": [
-              "lowercase",
-              "mynGram"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "sg_search_analyzer": {
-            "filter": [
-              "standard",
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          }
-        }
-      }
-    },
-    "number_of_shards": 5
-  },
-  "aliases": {
-    "lp_suggestion_index_alias": {}
-  },
-  "mappings": {
-    "sg": {
-      "dynamic_templates": [
-        {
-          "longs": {
-            "mapping": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "match_mapping_type": "long"
-          }
-        },
-        {
-          "booleans": {
-            "mapping": {
-              "type": "boolean",
-              "fields": {
-                "raw": {
-                  "type": "boolean"
-                }
-              }
-            },
-            "match_mapping_type": "boolean"
-          }
-        },
-        {
-          "doubles": {
-            "mapping": {
-              "type": "double",
-              "fields": {
-                "raw": {
-                  "type": "double"
-                }
-              }
-            },
-            "match_mapping_type": "double"
-          }
-        },
-        {
-          "dates": {
-            "mapping": {
-              "type": "date",
-              "fields": {
-                "raw": {
-                  "type": "date"
-                }
-              }
-            },
-            "match_mapping_type": "date"
-          }
-        },
-        {
-          "strings": {
-            "mapping": {
-              "copy_to": "all_fields",
-              "search_analyzer": "sg_search_analyzer",
-              "analyzer": "sg_index_analyzer",
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "analyzer": "keylower",
-                  "type": "text",
-                  "fielddata": true
-                }
-              }
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "properties": {
-        "all_fields": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "command": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "comment": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "comments": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "createdOn": {
-          "type": "date",
-          "fields": {
-            "raw": {
-              "type": "date",
-              "format": "strict_date_optional_time||epoch_millis"
-            }
-          },
-          "format": "strict_date_optional_time||epoch_millis"
-        },
-        "es_metadata_id": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "lastUpdatedBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "objectId": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "objectType": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "params": {
-          "properties": {
-            "Language": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "ageGroup": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "ageGrups": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "attributions": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "audience": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "code": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "concepts": {
-              "properties": {
-                "description": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "identifier": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "name": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "objectType": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "relation": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "relationName": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                }
-              }
-            },
-            "description": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "domain": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "domains": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "genre": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "gradeLevel": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "language": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "medium": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "methods": {
-              "properties": {
-                "description": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "identifier": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "name": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "objectType": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "relation": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "relationName": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                }
-              }
-            },
-            "name": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "subject": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "tags": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            },
-            "templateType": {
-              "type": "text",
-              "fields": {
-                "raw": {
-                  "type": "text",
-                  "fielddata": true,
-                  "analyzer": "keylower"
-                }
-              },
-              "copy_to": [
-                "all_fields"
-              ],
-              "analyzer": "sg_index_analyzer",
-              "search_analyzer": "sg_search_analyzer"
-            }
-          }
-        },
-        "request": {
-          "properties": {
-            "content": {
-              "properties": {
-                "comments": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                },
-                "status": {
-                  "type": "text",
-                  "fields": {
-                    "raw": {
-                      "type": "text",
-                      "fielddata": true,
-                      "analyzer": "keylower"
-                    }
-                  },
-                  "copy_to": [
-                    "all_fields"
-                  ],
-                  "analyzer": "sg_index_analyzer",
-                  "search_analyzer": "sg_search_analyzer"
-                }
-              }
-            }
-          }
-        },
-        "status": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "suggestedBy": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        },
-        "suggestion_id": {
-          "type": "text",
-          "fields": {
-            "raw": {
-              "type": "text",
-              "fielddata": true,
-              "analyzer": "keylower"
-            }
-          },
-          "copy_to": [
-            "all_fields"
-          ],
-          "analyzer": "sg_index_analyzer",
-          "search_analyzer": "sg_search_analyzer"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/files/templates/summary.json b/ansible/roles/es6/files/templates/summary.json
deleted file mode 100644
index 016f450b7e..0000000000
--- a/ansible/roles/es6/files/templates/summary.json
+++ /dev/null
@@ -1,802 +0,0 @@
-{
-  "index_patterns": "summary-*",
-  "settings": {
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "events": {
-      "dynamic": false,
-      "properties": {
-        "eid": {
-          "type": "keyword"
-        },
-        "ets": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "mid": {
-          "type": "keyword"
-        },
-        "uid": {
-          "type": "keyword"
-        },
-        "ts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "syncts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "ver": {
-          "type": "keyword"
-        },
-        "channel": {
-          "type": "keyword"
-        },
-        "context":{
-          "properties": {
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "model": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "granularity": {
-              "type": "keyword"
-            },
-            "date_range": {
-              "properties": {
-                "from": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "to": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                }
-              }
-            }
-          }
-        },
-        "dimensions": {
-          "properties": {
-            "did": {
-              "type": "keyword"
-            },
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "pid": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "gdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "sid": {
-              "type": "keyword"
-            },
-            "channel": {
-              "type": "keyword"
-            },
-            "group_user": {
-              "type": "boolean"
-            },
-            "anonymous_user": {
-              "type": "boolean"
-            },
-            "uid": {
-              "type": "keyword"
-            },
-            "ss_mid": {
-              "type": "keyword"
-            },
-            "tag": {
-              "type": "keyword"
-            },
-            "period": {
-              "type": "integer"
-            },
-            "content_id": {
-              "type": "keyword"
-            },
-            "item_id": {
-              "type": "keyword"
-            },
-            "author_id": {
-              "type": "keyword"
-            },
-            "loc": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "mode": {
-              "type": "keyword"
-            },
-            "context_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            },
-            "object_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            }
-          }
-        },
-        "edata": {
-          "properties": {
-            "eks": {
-              "properties": {
-                "total_users_count": {
-                  "type": "long"
-                },
-                "total_devices_count": {
-                  "type": "long"
-                },
-                "total_content_count": {
-                  "type": "long"
-                },
-                "content_ids": {
-                  "type": "keyword"
-                },
-                "device_ids": {
-                  "type": "keyword"
-                },
-                "user_ids": {
-                  "type": "keyword"
-                },
-                "contents": {
-                  "type": "keyword"
-                },
-                "unique_users": {
-                  "type": "keyword"
-                },
-                "avg_ts_session": {
-                  "type": "double"
-                },
-                "total_sessions": {
-                  "type": "long"
-                },
-                "avg_interactions_min": {
-                  "type": "double"
-                },
-                "total_interactions": {
-                  "type": "long"
-                },
-                "avg_pageviews": {
-                  "type": "double"
-                },
-                "total_ts": {
-                  "type": "double"
-                },
-                "time_diff": {
-                  "type": "double"
-                },
-                "time_spent": {
-                  "type": "double"
-                },
-                "interact_events_per_min": {
-                  "type": "double"
-                },
-                "interact_events_count": {
-                  "type": "long"
-                },
-                "total_pageviews_count": {
-                  "type": "long"
-                },
-                "page_views_count": {
-                  "type": "long"
-                },
-                "ce_visits": {
-                  "type": "long"
-                },
-                "start_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "end_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "syncDate": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "timeSpent": {
-                  "type": "double"
-                },
-                "interactEventsPerMin": {
-                  "type": "double"
-                },
-                "mimeType": {
-                  "type": "keyword"
-                },
-                "contentType": {
-                  "type": "keyword"
-                },
-                "timeDiff": {
-                  "type": "double"
-                },
-                "mode": {
-                  "type": "keyword"
-                },
-                "telemetryVersion": {
-                  "type": "keyword"
-                },
-                "noOfInteractEvents": {
-                  "type": "long"
-                },
-                "interruptTime": {
-                  "type": "double"
-                },
-                "contentCount": {
-                  "type": "long"
-                },
-                "content": {
-                  "type": "keyword"
-                },
-                "time_stamp": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "first_visit": {
-                  "type": "boolean"
-                },
-                "noOfAttempts": {
-                  "type": "integer"
-                },
-                "m_side_loads": {
-                  "type": "integer"
-                },
-                "m_downloads": {
-                  "type": "integer"
-                },
-                "m_avg_rating": {
-                  "type": "double"
-                },
-                "m_ratings": {
-                  "type": "nested",
-                  "properties": {
-                    "rating": {
-                      "type": "integer"
-                    },
-                    "time": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    }
-                  }
-                },
-                "m_comments": {
-                  "type": "object"
-                },
-                "ce_total_ts": {
-                  "type": "double"
-                },
-                "ce_percent_sessions": {
-                  "type": "double"
-                },
-                "ce_visits_count": {
-                  "type": "long"
-                },
-                "ce_total_visits": {
-                  "type": "long"
-                },
-                "ce_percent_ts": {
-                  "type": "double"
-                },
-                "avg_ts": {
-                  "type": "double"
-                },
-                "inc_res_count": {
-                  "type": "long"
-                },
-                "incorrect_res": {
-                  "type": "nested",
-                  "properties": {
-                    "resp": {
-                      "type": "keyword"
-                    },
-                    "mmc": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "total_count": {
-                  "type": "long"
-                },
-                "correct_res_count": {
-                  "type": "long"
-                },
-                "correct_res": {
-                  "type": "keyword"
-                },
-                "itemId": {
-                  "type": "keyword"
-                },
-                "exTimeSpent": {
-                  "type": "double"
-                },
-                "score": {
-                  "type": "integer"
-                },
-                "pass": {
-                  "type": "keyword"
-                },
-                "qtitle": {
-                  "type": "keyword"
-                },
-                "qdesc": {
-                  "type": "keyword"
-                },
-                "new_user_count": {
-                  "type": "long"
-                },
-                "anon_total_sessions": {
-                  "type": "long"
-                },
-                "anon_total_ts": {
-                  "type": "double"
-                },
-                "anon_avg_ts_session": {
-                  "type": "double"
-                },
-                "percent_new_users_count": {
-                  "type": "double"
-                },
-                "ce_total_sessions": {
-                  "type": "long"
-                },
-                "unique_users_count": {
-                  "type": "long"
-                },
-                "stageVisitCount": {
-                  "type": "long"
-                },
-                "interactEventsCount": {
-                  "type": "long"
-                },
-                "interactEvents": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    }
-                  }
-                },
-                "stageId": {
-                  "type": "keyword"
-                },
-                "visitCount": {
-                  "type": "long"
-                },
-                "num_downloads": {
-                  "type": "long"
-                },
-                "num_sideloads": {
-                  "type": "long"
-                },
-                "avg_depth": {
-                  "type": "double"
-                },
-                "publish_pipeline_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "state": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "events_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "eventsSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "screenSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "visitCount": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "activitySummary": {
-                  "type": "nested",
-                  "properties": {
-                    "actType": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "page_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "visit_count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "env_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "itemResponses": {
-                  "type": "nested",
-                  "properties": {
-                    "itemId": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "exTimeSpent": {
-                      "type": "double"
-                    },
-                    "score": {
-                      "type": "integer"
-                    },
-                    "time_stamp": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    },
-                    "pass": {
-                      "type": "keyword"
-                    },
-                    "qtitle": {
-                      "type": "keyword"
-                    },
-                    "qdesc": {
-                      "type": "keyword"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "etags": {
-          "properties": {
-            "app": {
-              "type": "keyword"
-            },
-            "partner": {
-              "type": "keyword"
-            },
-            "dims": {
-              "type": "keyword"
-            }
-          }
-        },
-        "geoip": {
-          "properties": {
-            "location": {
-              "type": "geo_point"
-            }
-          },
-          "type": "object"
-        },
-        "contentdata": {
-          "properties": {
-            "ageGroup": {
-              "type": "keyword"
-            },
-            "author": {
-              "type": "keyword"
-            },
-            "audience": {
-              "type": "keyword"
-            },
-            "code": {
-              "type": "keyword"
-            },
-            "collaborators": {
-              "type": "keyword"
-            },
-            "collections": {
-              "type": "keyword"
-            },
-            "concepts": {
-              "type": "keyword"
-            },
-            "contentType": {
-              "type": "keyword"
-            },
-            "curriculum": {
-              "type": "keyword"
-            },
-            "developer": {
-              "type": "keyword"
-            },
-            "domain": {
-              "type": "keyword"
-            },
-            "downloadUrl": {
-              "type": "keyword"
-            },
-            "downloads": {
-              "type": "long"
-            },
-            "edition": {
-              "type": "keyword"
-            },
-            "genre": {
-              "type": "keyword"
-            },
-            "gradeLevel": {
-              "type": "keyword"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "me_totalDevices": {
-              "type": "long"
-            },
-            "me_totalDownloads": {
-              "type": "long"
-            },
-            "me_totalInteractions": {
-              "type": "long"
-            },
-            "me_totalRatings": {
-              "type": "long"
-            },
-            "me_totalSessionsCount": {
-              "type": "long"
-            },
-            "me_totalSideloads": {
-              "type": "long"
-            },
-            "me_totalTimespent": {
-              "type": "long"
-            },
-            "me_totalUsage": {
-              "type": "long"
-            },
-            "medium": {
-              "type": "keyword"
-            },
-            "methods": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "popularity": {
-              "type": "long"
-            },
-            "portalOwner": {
-              "type": "keyword"
-            },
-            "publication": {
-              "type": "keyword"
-            },
-            "publisher": {
-              "type": "keyword"
-            },
-            "rating": {
-              "type": "long"
-            },
-            "size": {
-              "type": "long"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "subject": {
-              "type": "keyword"
-            },
-            "templateType": {
-              "type": "keyword"
-            },
-            "theme": {
-              "type": "keyword"
-            },
-            "words": {
-              "type": "keyword"
-            }
-          }
-        },
-        "itemdata": {
-          "properties": {
-            "concepts": {
-              "type": "keyword"
-            },
-            "createdBy": {
-              "type": "keyword"
-            },
-            "createdOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "language": {
-              "type": "keyword"
-            },
-            "lastUpdatedBy": {
-              "type": "keyword"
-            },
-            "lastUpdatedOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "media": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "num_answers": {
-              "type": "long"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "qlevel": {
-              "type": "keyword"
-            },
-            "question": {
-              "type": "keyword"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "template": {
-              "type": "keyword"
-            },
-            "title": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "long"
-            }
-          }
-        },
-        "ldata": {
-            "properties": {
-              "country": {
-                "type": "keyword"
-              },
-              "state": {
-                "type": "keyword"
-              },
-              "district": {
-                "type": "keyword"
-              },
-              "city": {
-                "type": "keyword"
-              }
-            }
-          },
-        "@version": {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
diff --git a/ansible/roles/es6/files/templates/summary_cumulative.json b/ansible/roles/es6/files/templates/summary_cumulative.json
deleted file mode 100644
index 99e01b8bdf..0000000000
--- a/ansible/roles/es6/files/templates/summary_cumulative.json
+++ /dev/null
@@ -1,802 +0,0 @@
-{
-  "index_patterns": "summary-cumulative-*",
-  "settings": {
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "events": {
-      "dynamic": false,
-      "properties": {
-        "eid": {
-          "type": "keyword"
-        },
-        "ets": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "mid": {
-          "type": "keyword"
-        },
-        "uid": {
-          "type": "keyword"
-        },
-        "ts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "syncts": {
-          "format": "strict_date_optional_time||epoch_millis",
-          "type": "date"
-        },
-        "ver": {
-          "type": "keyword"
-        },
-        "channel": {
-          "type": "keyword"
-        },
-        "context":{
-          "properties": {
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "model": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "granularity": {
-              "type": "keyword"
-            },
-            "date_range": {
-              "properties": {
-                "from": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "to": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                }
-              }
-            }
-          }
-        },
-        "dimensions": {
-          "properties": {
-            "did": {
-              "type": "keyword"
-            },
-            "pdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "pid": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "gdata": {
-              "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "ver": {
-                  "type": "keyword"
-                }
-              }
-            },
-            "sid": {
-              "type": "keyword"
-            },
-            "channel": {
-              "type": "keyword"
-            },
-            "group_user": {
-              "type": "boolean"
-            },
-            "anonymous_user": {
-              "type": "boolean"
-            },
-            "uid": {
-              "type": "keyword"
-            },
-            "ss_mid": {
-              "type": "keyword"
-            },
-            "tag": {
-              "type": "keyword"
-            },
-            "period": {
-              "type": "integer"
-            },
-            "content_id": {
-              "type": "keyword"
-            },
-            "item_id": {
-              "type": "keyword"
-            },
-            "author_id": {
-              "type": "keyword"
-            },
-            "loc": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "mode": {
-              "type": "keyword"
-            },
-            "context_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            },
-            "object_rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-            }
-          }
-        },
-        "edata": {
-          "properties": {
-            "eks": {
-              "properties": {
-                "total_users_count": {
-                  "type": "long"
-                },
-                "total_devices_count": {
-                  "type": "long"
-                },
-                "total_content_count": {
-                  "type": "long"
-                },
-                "content_ids": {
-                  "type": "keyword"
-                },
-                "device_ids": {
-                  "type": "keyword"
-                },
-                "user_ids": {
-                  "type": "keyword"
-                },
-                "contents": {
-                  "type": "keyword"
-                },
-                "unique_users": {
-                  "type": "keyword"
-                },
-                "avg_ts_session": {
-                  "type": "double"
-                },
-                "total_sessions": {
-                  "type": "long"
-                },
-                "avg_interactions_min": {
-                  "type": "double"
-                },
-                "total_interactions": {
-                  "type": "long"
-                },
-                "avg_pageviews": {
-                  "type": "double"
-                },
-                "total_ts": {
-                  "type": "double"
-                },
-                "time_diff": {
-                  "type": "double"
-                },
-                "time_spent": {
-                  "type": "double"
-                },
-                "interact_events_per_min": {
-                  "type": "double"
-                },
-                "interact_events_count": {
-                  "type": "long"
-                },
-                "total_pageviews_count": {
-                  "type": "long"
-                },
-                "page_views_count": {
-                  "type": "long"
-                },
-                "ce_visits": {
-                  "type": "long"
-                },
-                "start_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "end_time": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "syncDate": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "timeSpent": {
-                  "type": "double"
-                },
-                "interactEventsPerMin": {
-                  "type": "double"
-                },
-                "mimeType": {
-                  "type": "keyword"
-                },
-                "contentType": {
-                  "type": "keyword"
-                },
-                "timeDiff": {
-                  "type": "double"
-                },
-                "mode": {
-                  "type": "keyword"
-                },
-                "telemetryVersion": {
-                  "type": "keyword"
-                },
-                "noOfInteractEvents": {
-                  "type": "long"
-                },
-                "interruptTime": {
-                  "type": "double"
-                },
-                "contentCount": {
-                  "type": "long"
-                },
-                "content": {
-                  "type": "keyword"
-                },
-                "time_stamp": {
-                  "format": "strict_date_optional_time||epoch_millis",
-                  "type": "date"
-                },
-                "first_visit": {
-                  "type": "boolean"
-                },
-                "noOfAttempts": {
-                  "type": "integer"
-                },
-                "m_side_loads": {
-                  "type": "integer"
-                },
-                "m_downloads": {
-                  "type": "integer"
-                },
-                "m_avg_rating": {
-                  "type": "double"
-                },
-                "m_ratings": {
-                  "type": "nested",
-                  "properties": {
-                    "rating": {
-                      "type": "integer"
-                    },
-                    "time": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    }
-                  }
-                },
-                "m_comments": {
-                  "type": "object"
-                },
-                "ce_total_ts": {
-                  "type": "double"
-                },
-                "ce_percent_sessions": {
-                  "type": "double"
-                },
-                "ce_visits_count": {
-                  "type": "long"
-                },
-                "ce_total_visits": {
-                  "type": "long"
-                },
-                "ce_percent_ts": {
-                  "type": "double"
-                },
-                "avg_ts": {
-                  "type": "double"
-                },
-                "inc_res_count": {
-                  "type": "long"
-                },
-                "incorrect_res": {
-                  "type": "nested",
-                  "properties": {
-                    "resp": {
-                      "type": "keyword"
-                    },
-                    "mmc": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "total_count": {
-                  "type": "long"
-                },
-                "correct_res_count": {
-                  "type": "long"
-                },
-                "correct_res": {
-                  "type": "keyword"
-                },
-                "itemId": {
-                  "type": "keyword"
-                },
-                "exTimeSpent": {
-                  "type": "double"
-                },
-                "score": {
-                  "type": "integer"
-                },
-                "pass": {
-                  "type": "keyword"
-                },
-                "qtitle": {
-                  "type": "keyword"
-                },
-                "qdesc": {
-                  "type": "keyword"
-                },
-                "new_user_count": {
-                  "type": "long"
-                },
-                "anon_total_sessions": {
-                  "type": "long"
-                },
-                "anon_total_ts": {
-                  "type": "double"
-                },
-                "anon_avg_ts_session": {
-                  "type": "double"
-                },
-                "percent_new_users_count": {
-                  "type": "double"
-                },
-                "ce_total_sessions": {
-                  "type": "long"
-                },
-                "unique_users_count": {
-                  "type": "long"
-                },
-                "stageVisitCount": {
-                  "type": "long"
-                },
-                "interactEventsCount": {
-                  "type": "long"
-                },
-                "interactEvents": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    }
-                  }
-                },
-                "stageId": {
-                  "type": "keyword"
-                },
-                "visitCount": {
-                  "type": "long"
-                },
-                "num_downloads": {
-                  "type": "long"
-                },
-                "num_sideloads": {
-                  "type": "long"
-                },
-                "avg_depth": {
-                  "type": "double"
-                },
-                "publish_pipeline_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "state": {
-                      "type": "keyword"
-                    },
-                    "subtype": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "events_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "eventsSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "screenSummary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "visitCount": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "activitySummary": {
-                  "type": "nested",
-                  "properties": {
-                    "actType": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "page_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "type": {
-                      "type": "keyword"
-                    },
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "visit_count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "env_summary": {
-                  "type": "nested",
-                  "properties": {
-                    "env": {
-                      "type": "keyword"
-                    },
-                    "time_spent": {
-                      "type": "double"
-                    },
-                    "count": {
-                      "type": "long"
-                    }
-                  }
-                },
-                "itemResponses": {
-                  "type": "nested",
-                  "properties": {
-                    "itemId": {
-                      "type": "keyword"
-                    },
-                    "timeSpent": {
-                      "type": "double"
-                    },
-                    "exTimeSpent": {
-                      "type": "double"
-                    },
-                    "score": {
-                      "type": "integer"
-                    },
-                    "time_stamp": {
-                      "format": "strict_date_optional_time||epoch_millis",
-                      "type": "date"
-                    },
-                    "pass": {
-                      "type": "keyword"
-                    },
-                    "qtitle": {
-                      "type": "keyword"
-                    },
-                    "qdesc": {
-                      "type": "keyword"
-                    }
-                  }
-                }
-              }
-            }
-          }
-        },
-        "etags": {
-          "properties": {
-            "app": {
-              "type": "keyword"
-            },
-            "partner": {
-              "type": "keyword"
-            },
-            "dims": {
-              "type": "keyword"
-            }
-          }
-        },
-        "geoip": {
-          "properties": {
-            "location": {
-              "type": "geo_point"
-            }
-          },
-          "type": "object"
-        },
-        "contentdata": {
-          "properties": {
-            "ageGroup": {
-              "type": "keyword"
-            },
-            "author": {
-              "type": "keyword"
-            },
-            "audience": {
-              "type": "keyword"
-            },
-            "code": {
-              "type": "keyword"
-            },
-            "collaborators": {
-              "type": "keyword"
-            },
-            "collections": {
-              "type": "keyword"
-            },
-            "concepts": {
-              "type": "keyword"
-            },
-            "contentType": {
-              "type": "keyword"
-            },
-            "curriculum": {
-              "type": "keyword"
-            },
-            "developer": {
-              "type": "keyword"
-            },
-            "domain": {
-              "type": "keyword"
-            },
-            "downloadUrl": {
-              "type": "keyword"
-            },
-            "downloads": {
-              "type": "long"
-            },
-            "edition": {
-              "type": "keyword"
-            },
-            "genre": {
-              "type": "keyword"
-            },
-            "gradeLevel": {
-              "type": "keyword"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "me_totalDevices": {
-              "type": "long"
-            },
-            "me_totalDownloads": {
-              "type": "long"
-            },
-            "me_totalInteractions": {
-              "type": "long"
-            },
-            "me_totalRatings": {
-              "type": "long"
-            },
-            "me_totalSessionsCount": {
-              "type": "long"
-            },
-            "me_totalSideloads": {
-              "type": "long"
-            },
-            "me_totalTimespent": {
-              "type": "long"
-            },
-            "me_totalUsage": {
-              "type": "long"
-            },
-            "medium": {
-              "type": "keyword"
-            },
-            "methods": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "popularity": {
-              "type": "long"
-            },
-            "portalOwner": {
-              "type": "keyword"
-            },
-            "publication": {
-              "type": "keyword"
-            },
-            "publisher": {
-              "type": "keyword"
-            },
-            "rating": {
-              "type": "long"
-            },
-            "size": {
-              "type": "long"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "subject": {
-              "type": "keyword"
-            },
-            "templateType": {
-              "type": "keyword"
-            },
-            "theme": {
-              "type": "keyword"
-            },
-            "words": {
-              "type": "keyword"
-            }
-          }
-        },
-        "itemdata": {
-          "properties": {
-            "concepts": {
-              "type": "keyword"
-            },
-            "createdBy": {
-              "type": "keyword"
-            },
-            "createdOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "keywords": {
-              "type": "keyword"
-            },
-            "language": {
-              "type": "keyword"
-            },
-            "lastUpdatedBy": {
-              "type": "keyword"
-            },
-            "lastUpdatedOn": {
-              "format": "strict_date_optional_time||epoch_millis",
-              "type": "date"
-            },
-            "media": {
-              "type": "keyword"
-            },
-            "name": {
-              "type": "keyword"
-            },
-            "num_answers": {
-              "type": "long"
-            },
-            "owner": {
-              "type": "keyword"
-            },
-            "qlevel": {
-              "type": "keyword"
-            },
-            "question": {
-              "type": "keyword"
-            },
-            "source": {
-              "type": "keyword"
-            },
-            "status": {
-              "type": "keyword"
-            },
-            "template": {
-              "type": "keyword"
-            },
-            "title": {
-              "type": "keyword"
-            },
-            "type": {
-              "type": "keyword"
-            },
-            "version": {
-              "type": "long"
-            }
-          }
-        },
-        "ldata": {
-            "properties": {
-              "country": {
-                "type": "keyword"
-              },
-              "state": {
-                "type": "keyword"
-              },
-              "district": {
-                "type": "keyword"
-              },
-              "city": {
-                "type": "keyword"
-              }
-            }
-          },
-        "@version": {
-          "type": "keyword"
-        }
-      }
-    }
-  }
-}
diff --git a/ansible/roles/es6/files/templates/telemetry.json b/ansible/roles/es6/files/templates/telemetry.json
deleted file mode 100644
index afe9522328..0000000000
--- a/ansible/roles/es6/files/templates/telemetry.json
+++ /dev/null
@@ -1,430 +0,0 @@
-{
-  "index_patterns" : "telemetry-*",
-  "settings" : {
-    "number_of_shards" : 5
-  },
-  "mappings" : {
-    "events" : {
-      "dynamic": false,
-      "properties": {
-          "@timestamp": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "@version": {
-            "type": "keyword"
-          },
-          "eid": {
-            "type": "keyword"
-          },
-          "ets": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "mid": {
-            "type": "keyword"
-          },
-          "ts": {
-            "format": "strict_date_optional_time||epoch_millis",
-            "type": "date"
-          },
-          "ver": {
-            "type": "keyword"
-          },
-          "actor": {
-             "properties": {
-                "id": {
-                  "type": "keyword"
-                },
-                "type": {
-                  "type": "keyword"
-                }
-             }
-          },
-          "edata": {
-            "properties": {
-              "comments": {
-                "type": "keyword"
-              },
-              "correlationid": {
-                "type": "keyword"
-              },
-              "duration": {
-                "type": "double"
-              },
-              "data": {
-                "type": "keyword"
-              },
-              "dir": {
-                "type": "keyword"
-              },
-              "errtype": {
-                "type": "keyword"
-              },
-              "id": {
-                "type": "keyword"
-              },
-              "level": {
-                "type": "keyword"
-              },
-              "loc": {
-                "type": "keyword"
-              },
-              "message": {
-                "type": "keyword"
-              },
-              "mode": {
-                "type": "keyword"
-              },
-              "pass": {
-                "type": "keyword"
-              },
-              "prevstate": {
-                "type": "keyword"
-              },
-              "pageid": {
-                "type": "keyword"
-              },
-              "query": {
-                "type": "keyword"
-              },
-              "rating": {
-                "type": "double"
-              },
-              "score": {
-                "type": "double"
-              },
-              "size": {
-                "type": "double"
-              },
-              "state": {
-                "type": "keyword"
-              },
-              "subtype": {
-                "type": "keyword"
-              },
-              "type": {
-                "type": "keyword"
-              },
-              "uri": {
-                "type": "keyword"
-              },
-              "items": {
-                "type": "nested",
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "origin": {
-                    "properties": {
-                      "id": {
-                        "type": "keyword"
-                      },
-                      "type": {
-                        "type": "keyword"
-                      }
-                    }
-                  }
-                }
-              },
-              "item": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "maxscore": {
-                    "type": "long"
-                  },
-                  "exlength": {
-                    "type": "long"
-                  },
-                  "uri": {
-                    "type": "keyword"
-                  },
-                  "desc": {
-                    "type": "keyword"
-                  },
-                  "title": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "target": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "category": {
-                    "type": "keyword"
-                  },
-                  "parent": {
-                    "properties": {
-                      "id": {
-                        "type": "keyword"
-                      },
-                      "type": {
-                        "type": "keyword"
-                      }
-                    }
-                  }
-                }
-              },
-              "visits": {
-                "type": "nested",
-                "properties": {
-                  "objid": {
-                    "type": "keyword"
-                  },
-                  "objtype": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "plugin": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "category": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "object": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "subtype": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "uaspec": {
-                "properties": {
-                  "agent": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  },
-                  "system": {
-                    "type": "keyword"
-                  },
-                  "platform": {
-                    "type": "keyword"
-                  },
-                  "raw": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "dspec": {
-                "properties": {
-                   "camera": {
-                      "type": "keyword"
-                    },
-                    "cpu": {
-                      "type": "keyword"
-                    },
-                    "edisk": {
-                      "type": "double"
-                    },
-                    "id": {
-                      "type": "keyword"
-                    },
-                    "idisk": {
-                      "type": "double"
-                    },
-                    "make": {
-                      "type": "keyword"
-                    },
-                    "mem": {
-                      "type": "double"
-                    },
-                    "os": {
-                      "type": "keyword"
-                    },
-                    "scrn": {
-                      "type": "double"
-                    },
-                    "sims": {
-                      "type": "double"
-                    }
-                }
-              }
-            }
-          },
-          "context": {
-            "properties": {
-              "channel": {
-                "type": "keyword"
-              },
-              "env": {
-                "type": "keyword"
-              },
-              "sid": {
-                "type": "keyword"
-              },
-              "did": {
-                "type": "keyword"
-              },
-              "pdata": {
-                "properties": {
-                  "id": {
-                    "type": "keyword"
-                  },
-                  "pid": {
-                    "type": "keyword"
-                  },
-                  "ver": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-              },
-              "cdata": {
-                "type": "nested",
-                "properties": {
-                  "type": {
-                    "type": "keyword"
-                  },
-                  "id": {
-                    "type": "keyword"
-                  }
-                }
-              }
-            }
-          },
-          "object": {
-            "properties": {
-              "id": {
-                "type": "keyword"
-              },
-              "type": {
-                "type": "keyword"
-              },
-              "subtype": {
-                "type": "keyword"
-              },
-              "parentid": {
-                "type": "keyword"
-              },
-              "parenttype": {
-                "type": "keyword"
-              },
-              "ver": {
-                "type": "keyword"
-              },
-              "rollup": {
-                "properties": {
-                  "l1": {
-                    "type": "keyword"
-                  },
-                  "l2": {
-                    "type": "keyword"
-                  },
-                  "l3": {
-                    "type": "keyword"
-                  },
-                  "l4": {
-                    "type": "keyword"
-                  }
-                }
-              }
-            }
-          },
-          "ldata": {
-            "properties": {
-              "country": {
-                "type": "keyword"
-              },
-              "state": {
-                "type": "keyword"
-              },
-              "district": {
-                "type": "keyword"
-              },
-              "city": {
-                "type": "keyword"
-              }
-            }
-          },
-          "metadata": {
-            "properties": {
-              "source": {
-                "type": "keyword"
-              },
-              "index_name": {
-                "type": "keyword"
-              },
-              "index_type": {
-                "type": "keyword"
-              },
-              "source_eid": {
-                "type": "keyword"
-              },
-              "source_mid": {
-                "type": "keyword"
-              },
-              "pump": {
-                "type": "keyword"
-              }
-            }
-          },
-          "flags": {
-            "properties": {
-              "v2_converted": {
-                "type": "boolean"
-              },
-              "dd_processed": {
-                "type": "boolean"
-              },
-              "tv_processed": {
-                "type": "boolean"
-              },
-              "ldata_retrieved": {
-                "type": "boolean"
-              }
-            }
-          }
-       }
-    }
-  }
-}
diff --git a/ansible/roles/es6/files/templates/vocabulary.json b/ansible/roles/es6/files/templates/vocabulary.json
deleted file mode 100644
index d50cc7ec0e..0000000000
--- a/ansible/roles/es6/files/templates/vocabulary.json
+++ /dev/null
@@ -1,130 +0,0 @@
-{
-  "index_patterns": "vocabularyterm",
-  "settings": {
-    "index": {
-      "analysis": {
-        "filter": {
-          "mynGram": {
-            "token_chars": [
-              "letter",
-              "digit",
-              "whitespace",
-              "punctuation",
-              "symbol"
-            ],
-            "min_gram": "1",
-            "type": "edge_ngram",
-            "max_gram": "20"
-          }
-        },
-        "analyzer": {
-          "keylower": {
-            "filter": "lowercase",
-            "tokenizer": "keyword"
-          },
-          "vt_index_analyzer": {
-            "filter": [
-              "lowercase",
-              "mynGram"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          },
-          "vt_search_analyzer": {
-            "filter": [
-              "standard",
-              "lowercase"
-            ],
-            "type": "custom",
-            "tokenizer": "standard"
-          }
-        }
-      }
-    },
-    "number_of_shards": 5
-  },
-  "mappings": {
-    "vt": {
-      "dynamic_templates": [
-        {
-          "longs": {
-            "mapping": {
-              "type": "long",
-              "fields": {
-                "raw": {
-                  "type": "long"
-                }
-              }
-            },
-            "match_mapping_type": "long"
-          }
-        },
-        {
-          "booleans": {
-            "mapping": {
-              "type": "boolean",
-              "fields": {
-                "raw": {
-                  "type": "boolean"
-                }
-              }
-            },
-            "match_mapping_type": "boolean"
-          }
-        },
-        {
-          "doubles": {
-            "mapping": {
-              "type": "double",
-              "fields": {
-                "raw": {
-                  "type": "double"
-                }
-              }
-            },
-            "match_mapping_type": "double"
-          }
-        },
-        {
-          "dates": {
-            "mapping": {
-              "type": "date",
-              "fields": {
-                "raw": {
-                  "type": "date"
-                }
-              }
-            },
-            "match_mapping_type": "date"
-          }
-        },
-        {
-          "strings": {
-            "mapping": {
-              "type": "text",
-              "copy_to": "all_fields",
-              "analyzer": "vt_index_analyzer",
-              "search_analyzer": "vt_search_analyzer",
-              "fields": {
-              "raw": {
-                "type": "text",
-                "fielddata": true,
-                "analyzer": "keylower"
-                }
-              }
-            },
-            "match_mapping_type": "string"
-          }
-        }
-      ],
-      "properties": {
-        "all_fields": {
-          "type": "text",
-          "fielddata": true,
-          "analyzer": "vt_index_analyzer",
-          "search_analyzer": "vt_search_analyzer"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/ansible/roles/es6/filter_plugins/custom.py b/ansible/roles/es6/filter_plugins/custom.py
deleted file mode 100644
index 22177cdd19..0000000000
--- a/ansible/roles/es6/filter_plugins/custom.py
+++ /dev/null
@@ -1,57 +0,0 @@
-__author__ = 'dale mcdiarmid'
-
-import re
-import os.path
-from six import string_types
-
-def modify_list(values=[], pattern='', replacement='', ignorecase=False):
-    ''' Perform a `re.sub` on every item in the list'''
-    if ignorecase:
-        flags = re.I
-    else:
-        flags = 0
-    _re = re.compile(pattern, flags=flags)
-    return [_re.sub(replacement, value) for value in values]
-
-def append_to_list(values=[], suffix=''):
-    if isinstance(values, string_types):
-        values = values.split(',')
-    return [str(value+suffix) for value in values]
-
-def array_to_str(values=[],separator=','):
-    return separator.join(values)
-
-def extract_role_users(users={},exclude_users=[]):
-    role_users=[]
-    for user,details in users.iteritems():
-        if user not in exclude_users and "roles" in details:
-            for role in details["roles"]:
-                role_users.append(role+":"+user)
-    return role_users
-
-def filename(filename=''):
-    return os.path.splitext(os.path.basename(filename))[0]
-
-def remove_reserved(user_roles={}):
-    not_reserved = []
-    for user_role,details in user_roles.items():
-        if not "metadata" in details or not "_reserved" in details["metadata"] or not details["metadata"]["_reserved"]:
-            not_reserved.append(user_role)
-    return not_reserved
-
-def filter_reserved(users_role={}):
-    reserved = []
-    for user_role,details in users_role.items():
-        if "metadata" in details and "_reserved" in details["metadata"] and details["metadata"]["_reserved"]:
-            reserved.append(user_role)
-    return reserved
-
-class FilterModule(object):
-    def filters(self):
-        return {'modify_list': modify_list,
-        'append_to_list':append_to_list,
-        'filter_reserved':filter_reserved,
-        'array_to_str':array_to_str,
-        'extract_role_users':extract_role_users,
-        'remove_reserved':remove_reserved,
-        'filename':filename}
\ No newline at end of file
diff --git a/ansible/roles/es6/filter_plugins/custom.pyc b/ansible/roles/es6/filter_plugins/custom.pyc
deleted file mode 100644
index 771ec10df5..0000000000
Binary files a/ansible/roles/es6/filter_plugins/custom.pyc and /dev/null differ
diff --git a/ansible/roles/es6/handlers/main.yml b/ansible/roles/es6/handlers/main.yml
deleted file mode 100644
index d71397a395..0000000000
--- a/ansible/roles/es6/handlers/main.yml
+++ /dev/null
@@ -1,14 +0,0 @@
-
-- name: reload systemd configuration
-  become: yes
-  command: systemctl daemon-reload
-
-# Restart service and ensure it is enabled
-
-- name: restart elasticsearch
-  become: yes
-  service: name={{instance_init_script | basename}} state=restarted enabled=yes
-  when:
-    - es_restart_on_change
-    - es_start_service
-  register: es_restarted
diff --git a/ansible/roles/es6/meta/main.yml b/ansible/roles/es6/meta/main.yml
deleted file mode 100644
index aeecec7767..0000000000
--- a/ansible/roles/es6/meta/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
----
-
-allow_duplicates: yes
-
-galaxy_info:
-  author: Robin Clarke, Jakob Reiter, Dale McDiarmid
-  description: Elasticsearch for Linux
-  company: "Elastic.co"
-  license: "license (Apache)"
-  min_ansible_version: 2.3.2
-  platforms:
-  - name: EL
-    versions:
-    - 6
-    - 7
-  - name: Debian
-    versions:
-    - all
-  - name: Ubuntu
-    versions:
-    - all
-  categories:
-    - system
-
-dependencies: []
diff --git a/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml b/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml
deleted file mode 100644
index d9fdd698bc..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-Debian-version-lock.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-- name: Debian - hold elasticsearch version
-  become: yes
-  command: apt-mark hold elasticsearch
-  register: hold_elasticsearch_result
-  changed_when: "hold_elasticsearch_result.stdout != 'elasticsearch was already set on hold.'"
diff --git a/ansible/roles/es6/tasks/elasticsearch-Debian.yml b/ansible/roles/es6/tasks/elasticsearch-Debian.yml
deleted file mode 100644
index ebaaa61744..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-Debian.yml
+++ /dev/null
@@ -1,48 +0,0 @@
----
-
-- name: set fact force_install to no
-  set_fact: force_install=no
-
-- name: set fact force_install to yes
-  set_fact: force_install=yes
-  when: es_allow_downgrades
-
-- name: Debian - Install apt-transport-https to support https APT downloads
-  become: yes
-  apt: name=apt-transport-https state=present
-  when: es_use_repository
-
-- name: Debian - Add Elasticsearch repository key
-  become: yes
-  apt_key: url="{{ es_apt_key }}" state=present
-  when: es_use_repository and es_apt_key
-
-- name: Debian - Add elasticsearch repository
-  become: yes
-  apt_repository: repo={{ item.repo }} state={{ item.state}}
-  with_items:
-    - { repo: "{{ es_apt_url_old }}", state: "absent" }
-    - { repo: "{{ es_apt_url }}", state: "present" }
-  when: es_use_repository
-
-- name: Debian - Include versionlock
-  include: elasticsearch-Debian-version-lock.yml
-  when: es_version_lock
-
-- name: Debian - Ensure elasticsearch is installed
-  become: yes
-  apt: name=elasticsearch{% if es_version is defined and es_version != "" %}={{ es_version }}{% endif %} state=present force={{force_install}} allow_unauthenticated={{ 'no' if es_apt_key else 'yes' }} cache_valid_time=86400
-  when: es_use_repository
-  register: debian_elasticsearch_install_from_repo
-  notify: restart elasticsearch
-
-- name: Debian - Download elasticsearch from url
-  get_url: url={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.deb{% endif %} dest=/tmp/elasticsearch-{{ es_version }}.deb validate_certs=no
-  when: not es_use_repository
-
-- name: Debian - Ensure elasticsearch is installed from downloaded package
-  become: yes
-  apt: deb=/tmp/elasticsearch-{{ es_version }}.deb
-  when: not es_use_repository
-  register: elasticsearch_install_from_package
-  notify: restart elasticsearch
diff --git a/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml b/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml
deleted file mode 100644
index b5711a2f3b..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-RedHat-version-lock.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-- name: RedHat - install yum-version-lock
-  become: yes
-  yum: name=yum-plugin-versionlock state=present update_cache=yes
-- name: RedHat - lock elasticsearch version
-  become: yes
-  shell: yum versionlock delete 0:elasticsearch* ; yum versionlock add elasticsearch{% if es_version is defined and es_version != "" %}-{{ es_version }}{% endif %}
diff --git a/ansible/roles/es6/tasks/elasticsearch-RedHat.yml b/ansible/roles/es6/tasks/elasticsearch-RedHat.yml
deleted file mode 100644
index 588f0cec04..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-RedHat.yml
+++ /dev/null
@@ -1,31 +0,0 @@
----
-- name: Ensure libselinux-python on CentOS 6.x
-  become: yes
-  yum: name=libselinux-python state=present update_cache=yes
-  when: ( ansible_distribution == "CentOS" ) and ( ansible_distribution_major_version == "6" )
-
-- name: RedHat - add Elasticsearch repo
-  become: yes
-  template: src=elasticsearch.repo dest=/etc/yum.repos.d/elasticsearch-{{ es_major_version }}.repo
-  when: es_use_repository
-
-- name: RedHat - include versionlock
-  include: elasticsearch-RedHat-version-lock.yml
-  when: es_version_lock
-
-- name: RedHat - Install Elasticsearch
-  become: yes
-  yum: name=elasticsearch{% if es_version is defined and es_version != ""  %}-{{ es_version }}{% endif %} state=present update_cache=yes
-  when: es_use_repository
-  register: redhat_elasticsearch_install_from_repo
-  notify: restart elasticsearch
-  until: '"failed" not in redhat_elasticsearch_install_from_repo'
-  retries: 5
-  delay: 10
-
-- name: RedHat - Install Elasticsearch from url
-  become: yes
-  yum: name={% if es_custom_package_url is defined %}{{ es_custom_package_url }}{% else %}{{ es_package_url }}-{{ es_version }}.noarch.rpm{% endif %} state=present
-  when: not es_use_repository
-  register: elasticsearch_install_from_package
-  notify: restart elasticsearch
diff --git a/ansible/roles/es6/tasks/elasticsearch-config.yml b/ansible/roles/es6/tasks/elasticsearch-config.yml
deleted file mode 100644
index 987de718da..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-config.yml
+++ /dev/null
@@ -1,107 +0,0 @@
----
-# Configure Elasticsearch Node
-
-#Create required directories
-- name: Create Directories
-  become: yes
-  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
-  with_items:
-    - "{{pid_dir}}"
-    - "{{log_dir}}"
-    - "{{conf_dir}}"
-
-- name: Create Data Directories
-  become: yes
-  file: path={{ item }} state=directory owner={{ es_user }} group={{ es_group }}
-  with_items:
-    - "{{data_dirs}}"
-
-
-#Copy the config template
-- name: Copy Configuration File
-  become: yes
-  template: src=elasticsearch.yml.j2 dest={{conf_dir}}/elasticsearch.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  register: system_change
-  notify: restart elasticsearch
-
-#Copy the instance specific default file
-- name: Copy Default File for Instance
-  become: yes
-  template: src=elasticsearch.j2 dest={{instance_default_file}} mode=0644 force=yes
-  notify: restart elasticsearch
-
-#Copy the instance specific init file
-- name: Copy Debian Init File for Instance
-  become: yes
-  template: src=init/debian/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes
-  when: ansible_os_family == 'Debian' and not use_system_d
-  notify: restart elasticsearch
-
-#Copy the instance specific init file
-- name: Copy Redhat Init File for Instance
-  become: yes
-  template: src=init/redhat/elasticsearch.j2 dest={{instance_init_script}} mode=0755 force=yes
-  when: ansible_os_family == 'RedHat' and not use_system_d
-  notify: restart elasticsearch
-
-#Copy the systemd specific file if systemd is installed
-- name: Copy Systemd File for Instance
-  become: yes
-  template: src=systemd/elasticsearch.j2 dest={{instance_sysd_script}} mode=0644 force=yes
-  when: use_system_d
-  notify:
-  - reload systemd configuration
-  - restart elasticsearch
-
-#Copy the logging.yml
-- name: Copy log4j2.properties File for Instance
-  become: yes
-  template: src={{es_config_log4j2}} dest={{conf_dir}}/log4j2.properties owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  notify: restart elasticsearch
-
-- name: Copy jvm.options File for Instance
-  become: yes
-  template: src=jvm.options.j2 dest={{conf_dir}}/jvm.options owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  notify: restart elasticsearch
-
-#Clean up un-wanted package scripts to avoid confusion
-
-- name: Delete Default Init
-  become: yes
-  file: dest=/etc/init.d/elasticsearch state=absent
-
-- name: Create empty default environment file
-  become: yes
-  changed_when: False
-  copy:
-    dest: /etc/default/elasticsearch
-    content: ''
-  when: ansible_os_family == 'Debian'
-
-- name: Create empty default environment file
-  become: yes
-  changed_when: False
-  copy:
-    dest: /etc/sysconfig/elasticsearch
-    content: ''
-  when: ansible_os_family == 'RedHat'
-
-- name: Delete Default Sysconfig File
-  become: yes
-  file: dest="{{ sysd_script }}" state=absent
-
-- name: Delete Default Configuration File
-  become: yes
-  file: dest=/etc/elasticsearch/elasticsearch.yml state=absent
-
-- name: Delete Default Logging File
-  become: yes
-  file: dest=/etc/elasticsearch/logging.yml state=absent
-
-- name: Delete Default Logging File
-  become: yes
-  file: dest=/etc/elasticsearch/log4j2.properties state=absent
-
-- name: Delete Default JVM Options File
-  become: yes
-  file: dest=/etc/elasticsearch/jvm.options state=absent
diff --git a/ansible/roles/es6/tasks/elasticsearch-optional-user.yml b/ansible/roles/es6/tasks/elasticsearch-optional-user.yml
deleted file mode 100644
index d8f787e4b7..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-optional-user.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-#Add the elasticsearch user before installing from packages.
-- name: Ensure optional elasticsearch group is created with the correct id.
-  become: yes
-  #Restart if these change
-  notify: restart elasticsearch
-  group:
-    state: present
-    name: "{{ es_group }}"
-    system: yes
-    gid: "{{ es_group_id }}"
-
-- name: Ensure optional elasticsearch user is created with the correct id.
-  become: yes
-  #Restart if these change
-  notify: restart elasticsearch
-  user:
-    state: present
-    name: "{{ es_user }}"
-    comment: elasticsearch system user
-    system: yes
-    createhome: no
-    uid: "{{ es_user_id }}"
-    group: "{{ es_group }}"
diff --git a/ansible/roles/es6/tasks/elasticsearch-parameters.yml b/ansible/roles/es6/tasks/elasticsearch-parameters.yml
deleted file mode 100644
index 13c6472a3d..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-parameters.yml
+++ /dev/null
@@ -1,75 +0,0 @@
-# Check for mandatory parameters
-
-- name: fail when es_instance is not defined
-  fail: msg="es_instance_name must be specified and cannot be blank"
-  when: es_instance_name is not defined or es_instance_name == ''
-
-- name: fail when es_proxy_port is not defined or is blank
-  fail: msg="es_proxy_port must be specified and cannot be blank when es_proxy_host is defined"
-  when: (es_proxy_port is not defined or es_proxy_port == '') and (es_proxy_host is defined and es_proxy_host != '')
-
-- name: debug message
-  debug: msg="WARNING - It is recommended you specify the parameter 'http.port'"
-  when: es_config['http.port'] is not defined
-
-- name: debug message
-  debug: msg="WARNING - It is recommended you specify the parameter 'transport.tcp.port'"
-  when: es_config['transport.tcp.port'] is not defined
-
-- name: debug message
-  debug: msg="WARNING - It is recommended you specify the parameter 'discovery.zen.ping.unicast.hosts'"
-  when: es_config['discovery.zen.ping.unicast.hosts'] is not defined
-
-#If the user attempts to lock memory they must specify a heap size
-- name: fail when heap size is not specified when using memory lock
-  fail: msg="If locking memory with bootstrap.memory_lock a heap size must be specified"
-  when: es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True and es_heap_size is not defined
-
-#Check if working with security we have an es_api_basic_auth_username and es_api_basic_auth_username - otherwise any http calls wont work
-- name: fail when api credentials are not declared when using security
-  fail: msg="Enabling security requires an es_api_basic_auth_username and es_api_basic_auth_password to be provided to allow cluster operations"
-  when: es_enable_xpack and ("security" in es_xpack_features) and es_api_basic_auth_username is not defined and es_api_basic_auth_password is not defined
-
-- name: set fact file_reserved_users
-  set_fact: file_reserved_users={{ es_users.file.keys() | intersect (reserved_xpack_users) }}
-  when: es_users is defined and es_users.file is defined and (es_users.file.keys() | length > 0) and (es_users.file.keys() | intersect (reserved_xpack_users) | length > 0)
-
-- name: fail when changing users through file realm
-  fail:
-     msg: "ERROR: INVALID CONFIG - YOU CANNOT CHANGE RESERVED USERS THROUGH THE FILE REALM. THE FOLLOWING CANNOT BE CHANGED: {{file_reserved_users}}. USE THE NATIVE REALM."
-  when: file_reserved_users | default([]) | length > 0
-
-- name: set fact instance_default_file
-  set_fact: instance_default_file={{default_file | dirname}}/{{es_instance_name}}_{{default_file | basename}}
-- name: set fact instance_init_script
-  set_fact: instance_init_script={{init_script | dirname }}/{{es_instance_name}}_{{init_script | basename}}
-- name: set fact conf_dir
-  set_fact: conf_dir={{ es_conf_dir }}/{{es_instance_name}}
-- name: set fact m_lock_enabled
-  set_fact: m_lock_enabled={{ es_config['bootstrap.memory_lock'] is defined and es_config['bootstrap.memory_lock'] == True }}
-
-#TODO - if transport.host is not local maybe error on boostrap checks
-
-
-#Use systemd for the following distributions:
-#Ubuntu 15 and up
-#Debian 8 and up
-#Centos 7 and up
-#Relies on elasticsearch distribution installing a serviced script to determine whether one should be copied.
-
-- name: set fact use_system_d
-  set_fact: use_system_d={{(ansible_distribution == 'Debian' and ansible_distribution_version | version_compare('8', '>=')) or (ansible_distribution in ['RedHat','CentOS'] and ansible_distribution_version | version_compare('7', '>=')) or (ansible_distribution == 'Ubuntu' and ansible_distribution_version | version_compare('15', '>=')) }}
-
-- name: set fact instance_sysd_script
-  set_fact: instance_sysd_script={{sysd_script | dirname }}/{{es_instance_name}}_{{sysd_script | basename}}
-  when: use_system_d
-#For directories we also use the {{inventory_hostname}}-{{ es_instance_name }} - this helps if we have a shared SAN.
-
-- name: set fact instance_suffix
-  set_fact: instance_suffix={{inventory_hostname}}-{{ es_instance_name }}
-- name: set fact pid_dir
-  set_fact: pid_dir={{ es_pid_dir }}/{{instance_suffix}}
-- name: set fact log_dir
-  set_fact: log_dir={{ es_log_dir }}/{{instance_suffix}}
-- name: set fact log_dir
-  set_fact: data_dirs={{ es_data_dirs | append_to_list('/'+instance_suffix) }}
diff --git a/ansible/roles/es6/tasks/elasticsearch-plugins.yml b/ansible/roles/es6/tasks/elasticsearch-plugins.yml
deleted file mode 100644
index 5d4e2d8153..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-plugins.yml
+++ /dev/null
@@ -1,82 +0,0 @@
----
-
-# es_plugins_reinstall will be set to true if elasticsearch_install_from_repo.changed or elasticsearch_install_from_package.changed
-# i.e. we have changed ES version(or we have clean installation of ES), or if no plugins listed. Otherwise it is false and requires explicitly setting.
-- name: set fact es_plugins_reinstall to true
-  set_fact: es_plugins_reinstall=true
-  when: (((debian_elasticsearch_install_from_repo is defined and debian_elasticsearch_install_from_repo.changed) or (redhat_elasticsearch_install_from_repo is defined and redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) or es_plugins is not defined or es_plugins is none
-
-- name: set fact list_command
-  set_fact: list_command=""
-#If we are reinstalling all plugins, e.g. to a version change, we need to remove all plugins (inc. x-pack) to install any plugins. Otherwise we don't consider x-pack so the role stays idempotent.
-- name: set fact list_command check for x-pack
-  set_fact: list_command="| grep -vE 'x-pack'"
-  when: not es_plugins_reinstall
-
-#List currently installed plugins. We have to list the directories as the list commmand fails if the ES version is different than the plugin version.
-- name: Check installed elasticsearch plugins
-  become: yes
-  shell: "ls {{es_home}}/plugins {{list_command}}"
-  register: installed_plugins
-  changed_when: False
-  ignore_errors: yes
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-#if es_plugins_reinstall is set to true we remove ALL plugins
-- name: set fact plugins_to_remove to install_plugins.stdout_lines
-  set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | default([]) }}"
-  when: es_plugins_reinstall
-
-#if the plugins listed are different than those requested, we remove those installed but not listed in the config
-- name: set fact plugins_to_remove to delete plugins installed but not listed in es_plugins
-  set_fact: plugins_to_remove="{{ installed_plugins.stdout_lines | difference(es_plugins | json_query('[*].plugin')) | default([]) }}"
-  when: not es_plugins_reinstall
-
-#if es_plugins_reinstall is set to true we (re)install ALL plugins
-- name: set fact plugins_to_install to es_plugins
-  set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | default([]) }}"
-  when: es_plugins_reinstall
-
-#if the plugins listed are different than those requested, we install those not installed but listed in the config
-- name: set fact to plugins_to_install to those in es_config but not installed
-  set_fact: plugins_to_install="{{ es_plugins | json_query('[*].plugin') | difference(installed_plugins.stdout_lines) | default([]) }}"
-  when: not es_plugins_reinstall
-
-# This removes any currently installed plugins (to prevent errors when reinstalling)
-- name: Remove elasticsearch plugins
-  become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin remove {{item}} --silent"
-  ignore_errors: yes
-  with_items: "{{ plugins_to_remove | default([]) }}"
-  notify: restart elasticsearch
-  register: plugin_removed
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-- name: Install elasticsearch plugins
-  become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin install {{ item.plugin }} --batch --silent"
-  register: plugin_installed
-  failed_when: "'ERROR' in plugin_installed.stdout"
-  changed_when: plugin_installed.rc == 0
-  with_items: "{{ es_plugins }}"
-  when: item.plugin in plugins_to_install
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-    ES_JAVA_OPTS: "{% if item.proxy_host is defined and item.proxy_host != '' and item.proxy_port is defined and item.proxy_port != ''%} -Dhttp.proxyHost={{ item.proxy_host }} -Dhttp.proxyPort={{ item.proxy_port }} -Dhttps.proxyHost={{ item.proxy_host }} -Dhttps.proxyPort={{ item.proxy_port }}  {% elif es_proxy_host is defined and es_proxy_host != '' %} -Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }} {% endif %}"
-  until: plugin_installed.rc == 0
-  retries: 5
-  delay: 5
-
-#Set permissions on plugins directory
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
diff --git a/ansible/roles/es6/tasks/elasticsearch-scripts.yml b/ansible/roles/es6/tasks/elasticsearch-scripts.yml
deleted file mode 100644
index e38c3b4c4d..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-scripts.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-
-- name: set fact es_script_dir
-  set_fact: es_script_dir={{ es_conf_dir }}/{{es_instance_name}}
-  tags:
-      - always
-
-- name: set fact es_script_dir when path.scripts
-  set_fact: es_script_dir={{es_config['path.scripts']}}
-  when: es_config['path.scripts'] is defined
-  tags:
-      - always
-
-- name: Create script dir
-  become: yes
-  file: state=directory path={{ es_script_dir }} owner={{ es_user }} group={{ es_group }} recurse=yes
-
-- name: Copy default scripts to elasticsearch
-  become: yes
-  copy: src=scripts dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
-  when: es_scripts_fileglob is not defined
-
-- name: Copy scripts to elasticsearch
-  become: yes
-  copy: src={{ item }} dest={{ es_script_dir }} owner={{ es_user }} group={{ es_group }}
-  with_fileglob: "{{ es_scripts_fileglob | default('') }}"
diff --git a/ansible/roles/es6/tasks/elasticsearch-template.yml b/ansible/roles/es6/tasks/elasticsearch-template.yml
deleted file mode 100644
index 6059c7d727..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch-template.yml
+++ /dev/null
@@ -1,40 +0,0 @@
----
-- name: ensure templates dir is created
-  file:
-    path: /etc/elasticsearch/templates
-    state: directory
-    owner: "{{ es_user }}"
-    group: "{{ es_group }}"
-
-- name: Copy templates to elasticsearch
-  copy: src={{ item }} dest=/etc/elasticsearch/templates owner={{ es_user }} group={{ es_group }}
-  register: load_templates
-  with_fileglob:
-    - "{{ es_templates_fileglob | default('') }}"
-
-- name: Install templates without auth
-  uri:
-    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
-    method: PUT
-    status_code: 200
-    body_format: json
-    body: "{{ lookup('file', item) }}"
-  when: load_templates.changed and es_start_service and not es_enable_xpack or not es_xpack_features is defined or "security" not in es_xpack_features
-  with_fileglob:
-    - "{{ es_templates_fileglob | default('') }}"
-  run_once: True
-
-- name: Install templates with auth
-  uri:
-    url: "http://{{es_api_host}}:{{es_api_port}}/_template/{{item | filename}}"
-    method: PUT
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-    body_format: json
-    body: "{{ lookup('file', item) }}"
-  when: load_templates.changed and es_start_service and es_enable_xpack and es_xpack_features is defined and "security" in es_xpack_features
-  with_fileglob:
-    - "{{ es_templates_fileglob | default('') }}"
-  run_once: True
diff --git a/ansible/roles/es6/tasks/elasticsearch.yml b/ansible/roles/es6/tasks/elasticsearch.yml
deleted file mode 100644
index e2361d49bf..0000000000
--- a/ansible/roles/es6/tasks/elasticsearch.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-
-- name: Include optional user and group creation.
-  when: (es_user_id is defined) and (es_group_id is defined)
-  include: elasticsearch-optional-user.yml
-
-- name: Include specific Elasticsearch
-  include: elasticsearch-Debian.yml
-  when: ansible_os_family == 'Debian'
-
-- name: Include specific Elasticsearch
-  include: elasticsearch-RedHat.yml
-  when: ansible_os_family == 'RedHat'
diff --git a/ansible/roles/es6/tasks/java.yml b/ansible/roles/es6/tasks/java.yml
deleted file mode 100644
index a4ecac04a4..0000000000
--- a/ansible/roles/es6/tasks/java.yml
+++ /dev/null
@@ -1,51 +0,0 @@
----
-
-- name: set fact java_state to present
-  set_fact: java_state="present"
-
-- name: set fact java_state to latest
-  set_fact: java_state="latest"
-  when: update_java == true
-
-- name: RedHat - Ensure Java is installed
-  become: yes
-  yum: name={{ java }} state={{java_state}}
-  when: ansible_os_family == 'RedHat'
-
-- name: Get the installed java path
-  shell: "update-alternatives --display java | grep '^/' | awk '{print $1}' | grep 1.8.0"
-  register: java_full_path
-  failed_when: False
-  changed_when: False
-  when: ansible_os_family == 'RedHat'
-
-- name: correct java version selected
-  alternatives:
-    name: java
-    path: "{{ java_full_path.stdout }}"
-    link: /usr/bin/java
-  when: ansible_os_family == 'RedHat' and java_full_path is defined
-
-- name: Refresh java repo
-  become: yes
-  apt: update_cache=yes
-  changed_when: false
-  when: ansible_os_family == 'Debian'
-
-- name: Debian - Ensure Java is installed
-  become: yes
-  apt: name={{ java }} state={{java_state}}
-  when: ansible_os_family == 'Debian'
-
-- name: register open_jdk version
-  shell: java -version 2>&1 | grep OpenJDK
-  register: open_jdk
-  ignore_errors: yes
-  changed_when: false
-
-#https://github.com/docker-library/openjdk/issues/19 - ensures tests pass due to java 8 broken certs
-- name: refresh the java ca-certificates
-  become: yes
-  command: /var/lib/dpkg/info/ca-certificates-java.postinst configure
-  when: ansible_distribution == 'Ubuntu' and open_jdk.rc == 0
-  changed_when: false
diff --git a/ansible/roles/es6/tasks/main.yml b/ansible/roles/es6/tasks/main.yml
deleted file mode 100644
index bc6a1ba865..0000000000
--- a/ansible/roles/es6/tasks/main.yml
+++ /dev/null
@@ -1,84 +0,0 @@
----
-- name: os-specific vars
-  include_vars: "{{ansible_os_family}}.yml"
-  tags:
-      - always
-
-- name: check-set-parameters
-  include: elasticsearch-parameters.yml
-  tags:
-      - always
-
-# - name: include java.yml
-#   include: java.yml
-#   when: es_java_install
-#   tags:
-#       - java
-
-- name: include elasticsearch.yml
-  include: elasticsearch.yml
-  tags:
-      - install
-
-- name: include elasticsearch-config.yml
-  include: elasticsearch-config.yml
-  tags:
-      - config
-
-- name: include elasticsearch-scripts.yml
-  include: elasticsearch-scripts.yml
-  when: es_scripts
-  tags:
-      - scripts
-
-- name: include elasticsearch-plugins.yml
-  include: elasticsearch-plugins.yml
-  when: es_plugins is defined or es_plugins_reinstall
-  tags:
-      - plugins
-
-  #We always execute xpack as we may need to remove features
-- name: include xpack/elasticsearch-xpack.yml
-  include: xpack/elasticsearch-xpack.yml
-  tags:
-      - xpack
-
-- name: flush handlers
-  meta: flush_handlers
-
-- name: Make sure elasticsearch is started
-  service: name={{instance_init_script | basename}} state=started enabled=yes
-  when: es_start_service
-
-- name: Wait for elasticsearch to startup
-  wait_for: host={{es_api_host}} port={{es_api_port}} delay=5 connect_timeout=1
-  when: es_restarted is defined and es_restarted.changed and es_start_service
-
-- name: set fact manage_native_realm to false
-  set_fact: manage_native_realm=false
-
-- name: set fact manage_native_realm to true
-  set_fact: manage_native_realm=true
-  when: es_start_service and (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.native is defined) or (es_roles is defined and es_roles.native is defined))
-
-# If playbook runs too fast, Native commands could fail as the Native Realm is not yet up
-- name: Wait 15 seconds for the Native Relm to come up
-  pause: seconds=15
-  when: manage_native_realm
-
-- name: activate-license
-  include: ./xpack/security/elasticsearch-xpack-activation.yml
-  when: es_start_service and es_enable_xpack and es_xpack_license is defined and es_xpack_license != ''
-
-#perform security actions here now elasticsearch is started
-- name: include xpack/security/elasticsearch-security-native.yml
-  include: ./xpack/security/elasticsearch-security-native.yml
-  when: manage_native_realm
-
-#Templates done after restart - handled by flushing the handlers. e.g. suppose user removes security on a running node and doesn't specify es_api_basic_auth_username and es_api_basic_auth_password.  The templates will subsequently not be removed if we don't wait for the node to restart.
-#We also do after the native realm to ensure any changes are applied here first and its denf up.
-- name: include elasticsearch-template.yml
-  include: elasticsearch-template.yml
-  when: es_templates
-  tags:
-      - templates
\ No newline at end of file
diff --git a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml
deleted file mode 100644
index 522f8161fd..0000000000
--- a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack-install.yml
+++ /dev/null
@@ -1,68 +0,0 @@
----
-
-#Test if feature is installed
-- name: Test if x-pack is installed
-  shell: "{{es_home}}/bin/elasticsearch-plugin list | grep x-pack"
-  become: yes
-  register: x_pack_installed
-  changed_when: False
-  failed_when: "'ERROR' in x_pack_installed.stdout"
-  check_mode: no
-  ignore_errors: yes
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-
-#Remove X-Pack if installed and its not been requested or the ES version has changed
-- name: Remove x-pack plugin
-  become: yes
-  command: "{{es_home}}/bin/elasticsearch-plugin remove x-pack"
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: x_pack_installed.rc == 0 and (not es_enable_xpack or es_version_changed)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-
-#Install plugin if not installed, or the es version has changed (so removed above), and its been requested
-- name: Download x-pack from url
-  get_url: url={{ es_xpack_custom_url }} dest=/tmp/x-pack-{{ es_version }}.zip
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-
-- name: Install x-pack plugin from local
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch file:///tmp/x-pack-{{ es_version }}.zip
-  register: xpack_state
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-
-- name: Delete x-pack zip file
-  file: dest=/tmp/x-pack-{{ es_version }}.zip state=absent
-  when: es_xpack_custom_url is defined
-
-- name: Install x-pack plugin from elastic.co
-  become: yes
-  command: >
-    {{es_home}}/bin/elasticsearch-plugin install --silent --batch x-pack
-  register: xpack_state
-  failed_when: "'ERROR' in xpack_state.stdout"
-  changed_when: xpack_state.rc == 0
-  when: (x_pack_installed.rc == 1 or es_version_changed) and (es_enable_xpack and es_xpack_custom_url is not defined)
-  notify: restart elasticsearch
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_INCLUDE: "{{ instance_default_file }}"
-    ES_JAVA_OPTS: "{% if es_proxy_host is defined and es_proxy_host != '' %}-Dhttp.proxyHost={{ es_proxy_host }} -Dhttp.proxyPort={{ es_proxy_port }} -Dhttps.proxyHost={{ es_proxy_host }} -Dhttps.proxyPort={{ es_proxy_port }}{% endif %}"
diff --git a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml b/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml
deleted file mode 100644
index ec82a68a6f..0000000000
--- a/ansible/roles/es6/tasks/xpack/elasticsearch-xpack.yml
+++ /dev/null
@@ -1,22 +0,0 @@
----
-
-- name: set fact es_version_changed
-  set_fact: es_version_changed={{ ((elasticsearch_install_from_package is defined and (debian_elasticsearch_install_from_repo.changed or redhat_elasticsearch_install_from_repo.changed)) or (elasticsearch_install_from_package is defined and elasticsearch_install_from_package.changed)) }}
-
-- name: include elasticsearch-xpack-install.yml
-  include: elasticsearch-xpack-install.yml
-
-#Security configuration
-- name: include security/elasticsearch-security.yml
-  include: security/elasticsearch-security.yml
-
-#Add any feature specific configuration here
-- name: Set Plugin Directory Permissions
-  become: yes
-  file: state=directory path={{ es_home }}/plugins owner={{ es_user }} group={{ es_group }} recurse=yes
-
-#Make sure elasticsearch.keystore has correct Permissions
-- name: Set elasticsearch.keystore Permissions
-  become: yes
-  file: state=file path={{ conf_dir }}/elasticsearch.keystore owner={{ es_user }} group={{ es_group }}
-  when: es_enable_xpack and "security" in es_xpack_features and (es_version | version_compare('6.0.0', '>'))
diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml
deleted file mode 100644
index 8d6f878995..0000000000
--- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-file.yml
+++ /dev/null
@@ -1,80 +0,0 @@
----
-- name: set fact manage_file_users
-  set_fact: manage_file_users=es_users is defined and es_users.file is defined and es_users.file.keys() | length > 0
-
-#List current users
-- name: List Users
-  become: yes
-  shell: cat {{conf_dir}}/x-pack/users | awk -F':' '{print $1}'
-  register: current_file_users
-  when: manage_file_users
-  changed_when: False
-
-- name: set fact users_to_remove
-  set_fact: users_to_remove={{ current_file_users.stdout_lines | difference (es_users.file.keys()) }}
-  when: manage_file_users
-
-#Remove users
-- name: Remove Users
-  become: yes
-  command: >
-    {{es_home}}/bin/x-pack/users userdel {{item}}
-  with_items: "{{users_to_remove | default([])}}"
-  when: manage_file_users
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_HOME: "{{es_home}}"
-
-- name: set fact users_to_add
-  set_fact: users_to_add={{ es_users.file.keys() | difference (current_file_users.stdout_lines) }}
-  when: manage_file_users
-
-#Add users
-- name: Add Users
-  become: yes
-  command: >
-    {{es_home}}/bin/x-pack/users useradd {{item}} -p {{es_users.file[item].password}}
-  with_items: "{{ users_to_add | default([]) }}"
-  when: manage_file_users
-  no_log: True
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_HOME: "{{es_home}}"
-
-#Set passwords for all users declared - Required as the useradd will not change existing user passwords
-- name: Set User Passwords
-  become: yes
-  command: >
-    {{es_home}}/bin/x-pack/users passwd {{ item }} -p {{es_users.file[item].password}}
-  with_items: "{{ es_users.file.keys() | default([]) }}"
-  when: manage_file_users
-  #Currently no easy way to figure out if the password has changed or to know what it currently is so we can skip.
-  changed_when: False
-  no_log: True
-  environment:
-    CONF_DIR: "{{ conf_dir }}"
-    ES_PATH_CONF: "{{ conf_dir }}"
-    ES_HOME: "{{es_home}}"
-
-- name: set fact users_roles
-  set_fact: users_roles={{es_users.file | extract_role_users () }}
-  when: manage_file_users
-
-#Copy Roles files
-- name: Copy roles.yml File for Instance
-  become: yes
-  template: src=security/roles.yml.j2 dest={{conf_dir}}/x-pack/roles.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  when: es_roles is defined and es_roles.file is defined
-
-#Overwrite users_roles file
-- name: Copy User Roles
-  become: yes
-  template: src=security/users_roles.j2 dest={{conf_dir}}/x-pack/users_roles mode=0644 force=yes
-  when: manage_file_users and users_roles | length > 0
-
-#Set permission on security directory. E.g. if 2 nodes are installed on the same machine, the second node will not get the users file created at install, causing the files being created at es_users call and then having the wrong Permissions.
-- name: Set Security Directory Permissions Recursive
-  become: yes
-  file: state=directory path={{conf_dir}}/x-pack/ owner={{ es_user }} group={{ es_group }} recurse=yes
diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml
deleted file mode 100644
index 63024fadef..0000000000
--- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security-native.yml
+++ /dev/null
@@ -1,191 +0,0 @@
----
-- name: set fact change_api_password to false
-  set_fact: change_api_password=false
-
-- name: set fact manage_native_users to false
-  set_fact: manage_native_users=false
-
-- name: set fact manage_native_users to true
-  set_fact: manage_native_users=true
-  when: es_users is defined and es_users.native is defined and es_users.native.keys() | length > 0
-
-- name: set fact manage_native_role to false
-  set_fact: manage_native_roles=false
-
-- name: set fact manange_native_roles to true
-  set_fact: manage_native_roles=true
-  when: es_roles is defined and es_roles.native is defined and es_roles.native.keys() | length > 0
-
-#If the node has just has security installed it maybe either stopped or started 1. if stopped, we need to start to load native realms 2. if started, we need to restart to load
-
-#List current users
-- name: List Native Users
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user
-    method: GET
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-    status_code: 200
-  register: user_list_response
-  when: manage_native_users
-
-- name: set fact reserved_users equals user_list_response.json
-  set_fact: reserved_users={{ user_list_response.json | filter_reserved }}
-  when: manage_native_users
-
-#Current users not inc. those reserved
-- name: set fact current_users equals user_list_response.json.keys not including reserved
-  set_fact: current_users={{ user_list_response.json.keys() | difference (reserved_users) }}
-  when: manage_native_users
-
-#We are changing the es_api_basic_auth_username password, so we need to do it first and update the param
-- name: set fact native_users
-  set_fact: native_users={{ es_users.native }}
-  when: manage_native_users
-
-- name: set fact change_api_password to true
-  set_fact: change_api_password=true
-  when: manage_native_users and es_api_basic_auth_username in native_users and native_users[es_api_basic_auth_username].password is defined
-
-- name: Update API User Password
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{es_api_basic_auth_username}}/_password
-    method: POST
-    body_format: json
-    body: "{ \"password\":\"{{native_users[es_api_basic_auth_username].password}}\" }"
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: change_api_password
-
-- name: set fact es_api_basic_auth_password
-  set_fact: es_api_basic_auth_password={{native_users[es_api_basic_auth_username].password}}
-  when: change_api_password
-
-#Identify users that are present in ES but not declared and thus should be removed
-- name: set fact users_to_remove
-  set_fact: users_to_remove={{ current_users | difference ( native_users.keys() ) }}
-  when: manage_native_users
-
-#Delete all non required users NOT inc. reserved
-- name: Delete Native Users
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
-    method: DELETE
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: manage_native_users
-  with_items: "{{ users_to_remove | default([]) }}"
-
-- name: set fact users_to_ignore
-  set_fact: users_to_ignore={{ native_users.keys() | intersect (reserved_users) }}
-  when: manage_native_users
-
-- name: debug message
-  debug:
-    msg: "WARNING: YOU CAN ONLY CHANGE THE PASSWORD FOR RESERVED USERS IN THE NATIVE REALM. ANY ROLE CHANGES WILL BE IGNORED: {{users_to_ignore}}"
-  when: manage_native_users and users_to_ignore | length > 0
-
-#Update password on all reserved users
-- name: Update Reserved User Passwords
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}/_password
-    method: POST
-    body_format: json
-    body: "{ \"password\":\"{{native_users[item].password}}\" }"
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: native_users[item].password is defined
-  no_log: True
-  with_items: "{{ users_to_ignore | default([]) }}"
-
-- name: set fact users_to_modify
-  set_fact: users_to_modify={{ native_users.keys() | difference (reserved_users) }}
-  when: manage_native_users
-
-#Overwrite all other users NOT inc. those reserved
-- name: Update Non-Reserved Native User Details
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/user/{{item}}
-    method: POST
-    body_format: json
-    body: "{{ native_users[item] | to_json }}"
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: manage_native_users
-  no_log: True
-  with_items: "{{ users_to_modify | default([]) }}"
-
-## ROLE CHANGES
-
-#List current roles not. inc those reserved
-- name: List Native Roles
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role
-    method: GET
-    body_format: json
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-    status_code: 200
-  register: role_list_response
-  when: manage_native_roles
-
-- name: set fact reserved roles
-  set_fact: reserved_roles={{ role_list_response.json | filter_reserved }}
-  when: manage_native_roles
-
-- name: set fact current roles
-  set_fact: current_roles={{ role_list_response.json.keys() | difference (reserved_roles) }}
-  when: manage_native_roles
-
-- name: set fact roles to ignore
-  set_fact: roles_to_ignore={{ es_roles.native.keys() | intersect (reserved_roles) | default([]) }}
-  when: manage_native_roles
-
-- name: debug message
-  debug:
-    msg: "WARNING: YOU CANNOT CHANGE RESERVED ROLES. THE FOLLOWING WILL BE IGNORED: {{roles_to_ignore}}"
-  when: manage_native_roles and roles_to_ignore | length > 0
-
-- name: set fact roles_to_remove
-  set_fact: roles_to_remove={{ current_roles | difference ( es_roles.native.keys()  ) }}
-  when: manage_native_roles
-
-#Delete all non required roles NOT inc. reserved
-- name: Delete Native Roles
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
-    method: DELETE
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: manage_native_roles
-  with_items: "{{roles_to_remove | default([]) }}"
-
-- name: set fact roles_to_modify
-  set_fact: roles_to_modify={{ es_roles.native.keys() | difference (reserved_roles) }}
-  when: manage_native_roles
-
-#Update other roles - NOT inc. reserved roles
-- name: Update Native Roles
-  uri:
-    url: http://{{es_api_host}}:{{es_api_port}}/_xpack/security/role/{{item}}
-    method: POST
-    body_format: json
-    body: "{{ es_roles.native[item] | to_json}}"
-    status_code: 200
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    force_basic_auth: yes
-  when: manage_native_roles
-  with_items: "{{ roles_to_modify | default([]) }}"
diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml
deleted file mode 100644
index 14f4d20386..0000000000
--- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-security.yml
+++ /dev/null
@@ -1,60 +0,0 @@
----
-#Security specific configuration done here
-
-#TODO: 1. Skip users with no password defined or error 2. Passwords | length > 6
-
-#Ensure x-pack conf directory is created if necessary
-- name: Ensure x-pack conf directory exists (file)
-  file: path={{ conf_dir }}/x-pack state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when:
-    - es_enable_xpack and "security" in es_xpack_features
-    - (es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined) or (es_role_mapping is defined)
-
-#-----------------------------Create Bootstrap User-----------------------------------
-- name: Check if bootstrap password is set
-  command: >
-   {{es_home}}/bin/elasticsearch-keystore list
-  register: list_keystore
-  changed_when: False
-  environment:
-    ES_PATH_CONF: "{{ conf_dir }}"
-  when:
-    - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>'))
-
-- name: Create Bootstrap password for elastic user
-  shell: echo "{{es_api_basic_auth_password}}" | {{es_home}}/bin/elasticsearch-keystore add -x 'bootstrap.password'
-  when:
-    - (es_enable_xpack and "security" in es_xpack_features) and (es_version | version_compare('6.0.0', '>')) and es_api_basic_auth_username is defined and list_keystore is defined and es_api_basic_auth_username == 'elastic' and 'bootstrap.password' not in list_keystore.stdout_lines
-  environment:
-    ES_PATH_CONF: "{{ conf_dir }}"
-  no_log: true
-
-#-----------------------------FILE BASED REALM----------------------------------------
-
-- include: elasticsearch-security-file.yml
-  when: (es_enable_xpack and "security" in es_xpack_features) and ((es_users is defined and es_users.file is defined) or (es_roles is defined and es_roles.file is defined))
-
-#-----------------------------ROLE MAPPING ----------------------------------------
-
-#Copy Roles files
-- name: Copy role_mapping.yml File for Instance
-  become: yes
-  template: src=security/role_mapping.yml.j2 dest={{conf_dir}}/x-pack/role_mapping.yml owner={{ es_user }} group={{ es_group }} mode=0644 force=yes
-  when: es_role_mapping is defined
-
-#-----------------------------AUTH FILE----------------------------------------
-
-- name: Copy message auth key to elasticsearch
-  become: yes
-  copy: src={{ es_message_auth_file }} dest={{conf_dir}}/x-pack/system_key owner={{ es_user }} group={{ es_group }} mode=0600 force=yes
-  when: es_message_auth_file is defined
-
-#------------------------------------------------------------------------------------
-
-#Ensure security conf directory is created
-- name: Ensure security conf directory exists
-  become: yes
-  file: path={{ conf_dir }}/security state=directory owner={{ es_user }} group={{ es_group }}
-  changed_when: False
-  when: es_enable_xpack and "security" in es_xpack_features
diff --git a/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml b/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml
deleted file mode 100644
index cd72d6a7d2..0000000000
--- a/ansible/roles/es6/tasks/xpack/security/elasticsearch-xpack-activation.yml
+++ /dev/null
@@ -1,37 +0,0 @@
----
-
-- name: Activate ES license (without security authentication)
-  uri:
-    method: PUT
-    url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
-    body_format: json
-    body: "{{ es_xpack_license }}"
-    return_content: yes
-  register: license_activated
-  no_log: True
-  when: not "security" in es_xpack_features
-  failed_when: >
-    license_activated.status != 200 or
-    license_activated.json.license_status is not defined or
-    license_activated.json.license_status != 'valid'
-
-- name: Activate ES license (with security authentication)
-  uri:
-    method: PUT
-    url: "http://{{es_api_host}}:{{es_api_port}}/_xpack/license?acknowledge=true"
-    user: "{{es_api_basic_auth_username}}"
-    password: "{{es_api_basic_auth_password}}"
-    body_format: json
-    force_basic_auth: yes
-    body: "{{ es_xpack_license }}"
-    return_content: yes
-  register: license_activated
-  no_log: True
-  when: "'security' in es_xpack_features"
-  failed_when: >
-    license_activated.status != 200 or
-    license_activated.json.license_status is not defined or
-    license_activated.json.license_status != 'valid'
-
-- debug:
-    msg: "License: {{ license_activated }}"
diff --git a/ansible/roles/es6/templates/elasticsearch.j2 b/ansible/roles/es6/templates/elasticsearch.j2
deleted file mode 100644
index 5bf5746f93..0000000000
--- a/ansible/roles/es6/templates/elasticsearch.j2
+++ /dev/null
@@ -1,83 +0,0 @@
-################################
-# Elasticsearch
-################################
-
-# Elasticsearch home directory
-ES_HOME={{es_home}}
-
-# Elasticsearch Java path
-#JAVA_HOME=
-
-# Elasticsearch configuration directory
-CONF_DIR={{conf_dir}}
-ES_PATH_CONF={{conf_dir}}
-
-# Elasticsearch data directory
-DATA_DIR={{ data_dirs | array_to_str }}
-
-# Elasticsearch logs directory
-LOG_DIR={{log_dir}}
-
-# Elasticsearch PID directory
-PID_DIR={{pid_dir}}
-
-ES_JVM_OPTIONS={{conf_dir}}/jvm.options
-
-# Configure restart on package upgrade (true, every other setting will lead to not restarting)
-#ES_RESTART_ON_UPGRADE=true
-
-# Path to the GC log file
-#ES_GC_LOG_FILE=/var/log/elasticsearch/gc.log
-
-################################
-# Elasticsearch service
-################################
-
-# SysV init.d
-#
-# When executing the init script, this user will be used to run the elasticsearch service.
-# The default value is 'elasticsearch' and is declared in the init.d file.
-# Note that this setting is only used by the init script. If changed, make sure that
-# the configured user can read and write into the data, work, plugins and log directories.
-# For systemd service, the user is usually configured in file /usr/lib/systemd/system/elasticsearch.service
-ES_USER={{es_user}}
-ES_GROUP={{es_group}}
-
-# The number of seconds to wait before checking if Elasticsearch started successfully as a daemon process
-ES_STARTUP_SLEEP_TIME=5
-
-################################
-# System properties
-################################
-
-# Specifies the maximum file descriptor number that can be opened by this process
-# When using Systemd, this setting is ignored and the LimitNOFILE defined in
-# /usr/lib/systemd/system/elasticsearch.service takes precedence
-{% if es_max_open_files is defined %}
-#MAX_OPEN_FILES
-MAX_OPEN_FILES={{es_max_open_files}}
-{% endif %}
-
-# The maximum number of bytes of memory that may be locked into RAM
-# Set to "unlimited" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml
-# When using Systemd, the LimitMEMLOCK property must be set
-# in /usr/lib/systemd/system/elasticsearch.service
-#MAX_LOCKED_MEMORY=
-{% if m_lock_enabled %}
-MAX_LOCKED_MEMORY=unlimited
-{% endif %}
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-# When using Systemd, this setting is ignored and the 'vm.max_map_count'
-# property is set at boot time in /usr/lib/sysctl.d/elasticsearch.conf
-#MAX_MAP_COUNT=262144
-{% if es_max_map_count is defined %}
-MAX_MAP_COUNT={{es_max_map_count}}
-{% endif %}
-
-# Specifies the maximum number of threads that can be started.
-# Elasticsearch requires a minimum of 2048.
-{% if es_max_threads is defined %}
-MAX_THREADS={{ es_max_threads }}
-{% endif %}
diff --git a/ansible/roles/es6/templates/elasticsearch.repo b/ansible/roles/es6/templates/elasticsearch.repo
deleted file mode 100644
index b6299042bd..0000000000
--- a/ansible/roles/es6/templates/elasticsearch.repo
+++ /dev/null
@@ -1,11 +0,0 @@
-[elasticsearch-{{ es_major_version }}]
-name=Elasticsearch repository for {{ es_major_version }} packages
-baseurl=https://artifacts.elastic.co/packages/{{ es_major_version }}/yum
-gpgcheck=1
-gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
-enabled=1
-autorefresh=1
-type=rpm-md
-{% if es_proxy_host is defined and es_proxy_host != '' and es_proxy_port is defined %}
-proxy=http://{{ es_proxy_host }}:{{es_proxy_port}}
-{% endif %}
diff --git a/ansible/roles/es6/templates/elasticsearch.yml.j2 b/ansible/roles/es6/templates/elasticsearch.yml.j2
deleted file mode 100644
index dcf6f7aef1..0000000000
--- a/ansible/roles/es6/templates/elasticsearch.yml.j2
+++ /dev/null
@@ -1,52 +0,0 @@
-
-{% if es_config %}
-{{ es_config | to_nice_yaml }}
-{% endif %}
-
-{% if es_config['cluster.name'] is not defined %}
-cluster.name: elasticsearch
-{% endif %}
-
-{% if es_config['node.name'] is not defined %}
-node.name: {{inventory_hostname}}-{{es_instance_name}}
-{% endif %}
-
-#################################### Paths ####################################
-
-# Path to directory containing configuration (this file and logging.yml):
-
-{% if (es_version | version_compare('6.0.0', '<')) %}
-path.conf: {{ conf_dir }}
-{% endif %}
-
-path.data: {{ data_dirs | array_to_str }}
-
-path.logs: {{ log_dir }}
-
-{% if es_enable_xpack %}
-{% if not "security" in es_xpack_features %}
-xpack.security.enabled: false
-{% endif %}
-
-{% if not "monitoring" in es_xpack_features %}
-xpack.monitoring.enabled: false
-{% endif %}
-
-{% if not "alerting" in es_xpack_features %}
-xpack.watcher.enabled: false
-{% endif %}
-
-{% if not "ml" in es_xpack_features %}
-xpack.ml.enabled: false
-{% endif %}
-
-{% if not "graph" in es_xpack_features %}
-xpack.graph.enabled: false
-{% endif %}
-{% endif %}
-
-network.host: 0.0.0.0
-
-{% if es_remote_reindex is defined %}
-reindex.remote.whitelist: {{es_remote_host}}:9200
-{% endif %}
diff --git a/ansible/roles/es6/templates/init/debian/elasticsearch.j2 b/ansible/roles/es6/templates/init/debian/elasticsearch.j2
deleted file mode 100755
index efe2c37f6c..0000000000
--- a/ansible/roles/es6/templates/init/debian/elasticsearch.j2
+++ /dev/null
@@ -1,229 +0,0 @@
-#!/bin/bash
-#
-# /etc/init.d/elasticsearch -- startup script for Elasticsearch
-#
-### BEGIN INIT INFO
-# Provides:          elasticsearch
-# Required-Start:    $network $remote_fs $named
-# Required-Stop:     $network $remote_fs $named
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Starts elasticsearch
-# Description:       Starts elasticsearch using start-stop-daemon
-### END INIT INFO
-
-PATH=/bin:/usr/bin:/sbin:/usr/sbin
-NAME={{es_instance_name}}_{{default_file | basename}}
-{% if es_config['node.name'] is defined %}
-DESC="Elasticsearch Server - {{es_config['node.name']}}"
-{% else %}
-DESC="Elasticsearch Server - {{es_instance_name}}"
-{% endif %}
-
-DEFAULT=/etc/default/$NAME
-
-if [ `id -u` -ne 0 ]; then
-	echo "You need root privileges to run this script"
-	exit 1
-fi
-
-. /lib/lsb/init-functions
-if [ -r /etc/default/rcS ]; then
-	. /etc/default/rcS
-fi
-
-# The following variables can be overwritten in $DEFAULT
-
-# Run Elasticsearch as this user ID and group ID
-ES_USER={{es_user}}
-ES_GROUP={{es_group}}
-
-# Directory where the Elasticsearch binary distribution resides
-ES_HOME={{es_home}}
-
-# Maximum number of open files
-{% if es_max_open_files is defined %}
-MAX_OPEN_FILES={{es_max_open_files}}
-{% endif %}
-
-# Maximum amount of locked memory
-#MAX_LOCKED_MEMORY=
-{% if m_lock_enabled %}
-MAX_LOCKED_MEMORY=unlimited
-{% endif %}
-
-# Elasticsearch log directory
-LOG_DIR={{log_dir}}
-
-# Elasticsearch data directory
-DATA_DIR={{ data_dirs | array_to_str }}
-
-# Elasticsearch configuration directory
-CONF_DIR={{conf_dir}}
-ES_PATH_CONF={{ conf_dir }}
-
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-{% if es_max_map_count is defined %}
-MAX_MAP_COUNT={{es_max_map_count}}
-{% endif %}
-
-# Elasticsearch PID file directory
-PID_DIR={{pid_dir}}
-
-ES_JVM_OPTIONS="{{conf_dir}}/jvm.options"
-
-# End of variables that can be overwritten in $DEFAULT
-
-# overwrite settings from default file
-if [ -f "$DEFAULT" ]; then
-	. "$DEFAULT"
-fi
-
-# CONF_FILE setting was removed
-if [ ! -z "$CONF_FILE" ]; then
-    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
-    exit 1
-fi
-
-if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
-    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
-fi
-
-# Define other required variables
-PID_FILE="$PID_DIR/$NAME.pid"
-DAEMON=$ES_HOME/bin/elasticsearch
-{% if (es_version | version_compare('6.0.0', '<')) %}
-DAEMON_OPTS="-d -p $PID_FILE -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR"
-{% else %}
-DAEMON_OPTS="-d -p $PID_FILE"
-{% endif %}
-
-export ES_JAVA_OPTS
-export JAVA_HOME
-export ES_INCLUDE
-export ES_JVM_OPTIONS
-export ES_PATH_CONF
-
-# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
-if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
-if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
-if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
-if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
-if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
-if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
-if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
-if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
-
-# Check DAEMON exists
-if [ ! -x "$DAEMON" ]; then
-        echo "The elasticsearch startup script does not exists or it is not executable, tried: $DAEMON"
-        exit 1
-fi
-
-checkJava() {
-	if [ -x "$JAVA_HOME/bin/java" ]; then
-		JAVA="$JAVA_HOME/bin/java"
-	else
-		JAVA=`which java`
-	fi
-
-	if [ ! -x "$JAVA" ]; then
-		echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
-		exit 1
-	fi
-}
-
-case "$1" in
-  start)
-	checkJava
-
-	log_daemon_msg "Starting $DESC"
-
-	pid=`pidofproc -p $PID_FILE elasticsearch`
-	if [ -n "$pid" ] ; then
-		log_begin_msg "Already running."
-		log_end_msg 0
-		exit 0
-	fi
-
-	# Ensure that the PID_DIR exists (it is cleaned at OS startup time)
-	if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
-		mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
-	fi
-	if [ -n "$PID_FILE" ] && [ ! -e "$PID_FILE" ]; then
-		touch "$PID_FILE" && chown "$ES_USER":"$ES_GROUP" "$PID_FILE"
-	fi
-
-	if [ -n "$MAX_OPEN_FILES" ]; then
-		ulimit -n $MAX_OPEN_FILES
-	fi
-
-	if [ -n "$MAX_LOCKED_MEMORY" ]; then
-		ulimit -l $MAX_LOCKED_MEMORY
-	fi
-
-	if [ -n "$MAX_THREADS" ]; then
-		ulimit -u $MAX_THREADS
-	fi
-
-	if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
-		sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
-	fi
-
-	# Start Daemon
-	start-stop-daemon -d $ES_HOME --start --user "$ES_USER" -c "$ES_USER" --pidfile "$PID_FILE" --exec $DAEMON -- $DAEMON_OPTS
-	return=$?
-	if [ $return -eq 0 ]; then
-		i=0
-		timeout={{es_debian_startup_timeout}}
-		# Wait for the process to be properly started before exiting
-		until { kill -0 `cat "$PID_FILE"`; } >/dev/null 2>&1
-		do
-			sleep 1
-			i=$(($i + 1))
-			if [ $i -gt $timeout ]; then
-				log_end_msg 1
-				exit 1
-			fi
-		done
-	fi
-	log_end_msg $return
-	exit $return
-	;;
-  stop)
-	log_daemon_msg "Stopping $DESC"
-
-	if [ -f "$PID_FILE" ]; then
-		start-stop-daemon --stop --pidfile "$PID_FILE" \
-			--user "$ES_USER" \
-			--quiet \
-			--retry forever/TERM/20 > /dev/null
-		if [ $? -eq 1 ]; then
-			log_progress_msg "$DESC is not running but pid file exists, cleaning up"
-		elif [ $? -eq 3 ]; then
-			PID="`cat $PID_FILE`"
-			log_failure_msg "Failed to stop $DESC (pid $PID)"
-			exit 1
-		fi
-		rm -f "$PID_FILE"
-	else
-		log_progress_msg "(not running)"
-	fi
-	log_end_msg 0
-	;;
-  status)
-	status_of_proc -p $PID_FILE elasticsearch elasticsearch && exit 0 || exit $?
-	;;
-  restart|force-reload)
-	if [ -f "$PID_FILE" ]; then
-		$0 stop
-	fi
-	$0 start
-	;;
-  *)
-	log_success_msg "Usage: $0 {start|stop|restart|force-reload|status}"
-	exit 1
-	;;
-esac
-
-exit 0
diff --git a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2 b/ansible/roles/es6/templates/init/redhat/elasticsearch.j2
deleted file mode 100755
index c993e14365..0000000000
--- a/ansible/roles/es6/templates/init/redhat/elasticsearch.j2
+++ /dev/null
@@ -1,217 +0,0 @@
-#!/bin/bash
-#
-# elasticsearch <summary>
-#
-# chkconfig:   2345 80 20
-# description: Starts and stops a single elasticsearch instance on this system
-#
-
-### BEGIN INIT INFO
-# Provides: Elasticsearch
-# Required-Start: $network $named
-# Required-Stop: $network $named
-# Default-Start: 2 3 4 5
-# Default-Stop: 0 1 6
-# Short-Description: This service manages the elasticsearch daemon
-# Description: Elasticsearch is a very scalable, schema-free and high-performance search solution supporting multi-tenancy and near realtime search.
-### END INIT INFO
-
-#
-# init.d / servicectl compatibility (openSUSE)
-#
-if [ -f /etc/rc.status ]; then
-    . /etc/rc.status
-    rc_reset
-fi
-
-#
-# Source function library.
-#
-if [ -f /etc/rc.d/init.d/functions ]; then
-    . /etc/rc.d/init.d/functions
-fi
-
-# Sets the default values for elasticsearch variables used in this script
-ES_USER="{{es_user}}"
-ES_GROUP="{{es_group}}"
-ES_HOME="{{es_home}}"
-{% if es_max_open_files is defined %}
-MAX_OPEN_FILES={{es_max_open_files}}
-{% endif %}
-# Maximum number of VMA (Virtual Memory Areas) a process can own
-{% if es_max_map_count is defined %}
-MAX_MAP_COUNT={{es_max_map_count}}
-{% endif %}
-
-LOG_DIR="{{log_dir}}"
-DATA_DIR={{ data_dirs | array_to_str }}
-CONF_DIR="{{conf_dir}}"
-ES_PATH_CONF="{{ conf_dir }}"
-
-PID_DIR="{{pid_dir}}"
-
-# Source the default env file
-ES_ENV_FILE="{{instance_default_file}}"
-if [ -f "$ES_ENV_FILE" ]; then
-    . "$ES_ENV_FILE"
-fi
-
-if [ "$ES_USER" != "elasticsearch" ] || [ "$ES_GROUP" != "elasticsearch" ]; then
-    echo "WARNING: ES_USER and ES_GROUP are deprecated and will be removed in the next major version of Elasticsearch, got: [$ES_USER:$ES_GROUP]"
-fi
-
-# CONF_FILE setting was removed
-if [ ! -z "$CONF_FILE" ]; then
-    echo "CONF_FILE setting is no longer supported. elasticsearch.yml must be placed in the config directory and cannot be renamed."
-    exit 1
-fi
-
-exec="$ES_HOME/bin/elasticsearch"
-prog="{{es_instance_name}}_{{default_file | basename}}"
-pidfile="$PID_DIR/${prog}.pid"
-
-export ES_JAVA_OPTS
-export JAVA_HOME
-export ES_INCLUDE
-export ES_JVM_OPTIONS
-export ES_STARTUP_SLEEP_TIME
-export ES_PATH_CONF
-
-# export unsupported variables so bin/elasticsearch can reject them and inform the user these are unsupported
-if test -n "$ES_MIN_MEM"; then export ES_MIN_MEM; fi
-if test -n "$ES_MAX_MEM"; then export ES_MAX_MEM; fi
-if test -n "$ES_HEAP_SIZE"; then export ES_HEAP_SIZE; fi
-if test -n "$ES_HEAP_NEWSIZE"; then export ES_HEAP_NEWSIZE; fi
-if test -n "$ES_DIRECT_SIZE"; then export ES_DIRECT_SIZE; fi
-if test -n "$ES_USE_IPV4"; then export ES_USE_IPV4; fi
-if test -n "$ES_GC_OPTS"; then export ES_GC_OPTS; fi
-if test -n "$ES_GC_LOG_FILE"; then export ES_GC_LOG_FILE; fi
-
-lockfile=/var/lock/subsys/$prog
-
-# backwards compatibility for old config sysconfig files, pre 0.90.1
-if [ -n $USER ] && [ -z $ES_USER ] ; then
-   ES_USER=$USER
-fi
-
-if [ ! -x "$exec" ]; then
-    echo "The elasticsearch startup script does not exists or it is not executable, tried: $exec"
-    exit 1
-fi
-
-checkJava() {
-    if [ -x "$JAVA_HOME/bin/java" ]; then
-        JAVA="$JAVA_HOME/bin/java"
-    else
-        JAVA=`which java`
-    fi
-
-    if [ ! -x "$JAVA" ]; then
-        echo "Could not find any executable java binary. Please install java in your PATH or set JAVA_HOME"
-        exit 1
-    fi
-}
-
-start() {
-    checkJava
-    [ -x $exec ] || exit 5
-
-    if [ -n "$MAX_OPEN_FILES" ]; then
-        ulimit -n $MAX_OPEN_FILES
-    fi
-    if [ -n "$MAX_LOCKED_MEMORY" ]; then
-        ulimit -l $MAX_LOCKED_MEMORY
-    fi
-    if [ -n "$MAX_THREADS" ]; then
-        ulimit -u $MAX_THREADS
-    fi
-    if [ -n "$MAX_MAP_COUNT" -a -f /proc/sys/vm/max_map_count ]; then
-        sysctl -q -w vm.max_map_count=$MAX_MAP_COUNT
-    fi
-
-    # Ensure that the PID_DIR exists (it is cleaned at OS startup time)
-    if [ -n "$PID_DIR" ] && [ ! -e "$PID_DIR" ]; then
-        mkdir -p "$PID_DIR" && chown "$ES_USER":"$ES_GROUP" "$PID_DIR"
-    fi
-    if [ -n "$pidfile" ] && [ ! -e "$pidfile" ]; then
-        touch "$pidfile" && chown "$ES_USER":"$ES_GROUP" "$pidfile"
-    fi
-
-    cd $ES_HOME
-    echo -n $"Starting $prog: "
-    # if not running, start it up here, usually something like "daemon $exec"
-{% if (es_version | version_compare('6.0.0', '<')) %}
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d -Edefault.path.logs=$LOG_DIR -Edefault.path.data=$DATA_DIR -Edefault.path.conf=$CONF_DIR
-{% else %}
-    daemon --user $ES_USER --pidfile $pidfile $exec -p $pidfile -d
-{% endif %}
-    retval=$?
-    echo
-    [ $retval -eq 0 ] && touch $lockfile
-    return $retval
-}
-
-stop() {
-    echo -n $"Stopping $prog: "
-    # stop it here, often "killproc $prog"
-    killproc -p $pidfile -d 86400 $prog
-    retval=$?
-    echo
-    [ $retval -eq 0 ] && rm -f $lockfile
-    return $retval
-}
-
-restart() {
-    stop
-    start
-}
-
-reload() {
-    restart
-}
-
-force_reload() {
-    restart
-}
-
-rh_status() {
-    # run checks to determine if the service is running or use generic status
-    status -p $pidfile $prog
-}
-
-rh_status_q() {
-    rh_status >/dev/null 2>&1
-}
-
-
-case "$1" in
-    start)
-        rh_status_q && exit 0
-        $1
-        ;;
-    stop)
-        rh_status_q || exit 0
-        $1
-        ;;
-    restart)
-        $1
-        ;;
-    reload)
-        rh_status_q || exit 7
-        $1
-        ;;
-    force-reload)
-        force_reload
-        ;;
-    status)
-        rh_status
-        ;;
-    condrestart|try-restart)
-        rh_status_q || exit 0
-        restart
-        ;;
-    *)
-        echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload}"
-        exit 2
-esac
-exit $?
diff --git a/ansible/roles/es6/templates/jvm.options.j2 b/ansible/roles/es6/templates/jvm.options.j2
deleted file mode 100644
index ad30851a2b..0000000000
--- a/ansible/roles/es6/templates/jvm.options.j2
+++ /dev/null
@@ -1,118 +0,0 @@
-## JVM configuration
-
-################################################################
-## IMPORTANT: JVM heap size
-################################################################
-##
-## You should always set the min and max JVM heap
-## size to the same value. For example, to set
-## the heap to 4 GB, set:
-##
-## -Xms4g
-## -Xmx4g
-##
-## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
-## for more information
-##
-################################################################
-
-# Xms represents the initial size of total heap space
-# Xmx represents the maximum size of total heap space
-{% if es_heap_size is defined %}
--Xms{{ es_heap_size }}
--Xmx{{ es_heap_size }}
-{% else %}
--Xms2g
--Xmx2g
-{% endif %}
-
-################################################################
-## Expert settings
-################################################################
-##
-## All settings below this section are considered
-## expert settings. Don't tamper with them unless
-## you understand what you are doing
-##
-################################################################
-
-## GC configuration
--XX:+UseConcMarkSweepGC
--XX:CMSInitiatingOccupancyFraction=75
--XX:+UseCMSInitiatingOccupancyOnly
-
-## optimizations
-
-# pre-touch memory pages used by the JVM during initialization
--XX:+AlwaysPreTouch
-
-## basic
-
-# force the server VM
--server
-
-# set to headless, just in case
--Djava.awt.headless=true
-
-# ensure UTF-8 encoding by default (e.g. filenames)
--Dfile.encoding=UTF-8
-
-# use our provided JNA always versus the system one
--Djna.nosys=true
-
-# use old-style file permissions on JDK9
--Djdk.io.permissionsUseCanonicalPath=true
-
-# flags to configure Netty
--Dio.netty.noUnsafe=true
--Dio.netty.noKeySetOptimization=true
--Dio.netty.recycler.maxCapacityPerThread=0
-
-# log4j 2
--Dlog4j.shutdownHookEnabled=false
--Dlog4j2.disable.jmx=true
--Dlog4j.skipJansi=true
-
-## heap dumps
-
-# generate a heap dump when an allocation from the Java heap fails
-# heap dumps are created in the working directory of the JVM
--XX:+HeapDumpOnOutOfMemoryError
-
-# specify an alternative path for heap dumps
-# ensure the directory exists and has sufficient space
-#-XX:HeapDumpPath=${heap.dump.path}
-
-## GC logging
-
-#-XX:+PrintGCDetails
-#-XX:+PrintGCTimeStamps
-#-XX:+PrintGCDateStamps
-#-XX:+PrintClassHistogram
-#-XX:+PrintTenuringDistribution
-#-XX:+PrintGCApplicationStoppedTime
-
-# log GC status to a file with time stamps
-# ensure the directory exists
-#-Xloggc:${loggc}
-
-
-# By default, the GC log file will not rotate.
-# By uncommenting the lines below, the GC log file
-# will be rotated every 128MB at most 32 times.
-#-XX:+UseGCLogFileRotation
-#-XX:NumberOfGCLogFiles=32
-#-XX:GCLogFileSize=128M
-
-# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
-# If documents were already indexed with unquoted fields in a previous version
-# of Elasticsearch, some operations may throw errors.
-#
-# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
-# only for migration purposes.
-#-Delasticsearch.json.allow_unquoted_field_names=true
-{% if es_jvm_custom_parameters !='' %}
-{% for item in es_jvm_custom_parameters %}
-{{ item }}
-{% endfor %}
-{% endif %}
diff --git a/ansible/roles/es6/templates/log4j2.properties.j2 b/ansible/roles/es6/templates/log4j2.properties.j2
deleted file mode 100644
index 269be5290b..0000000000
--- a/ansible/roles/es6/templates/log4j2.properties.j2
+++ /dev/null
@@ -1,117 +0,0 @@
-status = error
-
-# log action execution errors for easier debugging
-logger.action.name = org.elasticsearch.action
-logger.action.level = debug
-
-appender.console.type = Console
-appender.console.name = console
-appender.console.layout.type = PatternLayout
-appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%m%n
-
-appender.rolling.type = RollingFile
-appender.rolling.name = rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.rolling.fileName = ${sys:es.logs}.log
-{% else %}
-appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log
-{% endif %}
-appender.rolling.layout.type = PatternLayout
-appender.rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.rolling.filePattern = ${sys:es.logs}-%d{yyyy-MM-dd}.log
-{% else %}
-appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz
-{% endif %}
-appender.rolling.policies.type = Policies
-appender.rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.rolling.policies.time.interval = 1
-appender.rolling.policies.time.modulate = true
-{% if (es_version | version_compare('6.0.0', '>')) %}
-appender.rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.rolling.policies.size.size = 128MB
-appender.rolling.strategy.type = DefaultRolloverStrategy
-appender.rolling.strategy.fileIndex = nomax
-appender.rolling.strategy.action.type = Delete
-appender.rolling.strategy.action.basepath = ${sys:es.logs.base_path}
-appender.rolling.strategy.action.condition.type = IfFileName
-appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-*
-appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize
-appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB
-{% endif %}
-rootLogger.level = info
-rootLogger.appenderRef.console.ref = console
-rootLogger.appenderRef.rolling.ref = rolling
-
-appender.deprecation_rolling.type = RollingFile
-appender.deprecation_rolling.name = deprecation_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.deprecation_rolling.fileName = ${sys:es.logs}_deprecation.log
-{% else %}
-appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log
-{% endif %}
-appender.deprecation_rolling.layout.type = PatternLayout
-appender.deprecation_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.deprecation_rolling.filePattern = ${sys:es.logs}_deprecation-%i.log.gz
-{% else %}
-appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.log.gz
-{% endif %}
-appender.deprecation_rolling.policies.type = Policies
-appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy
-appender.deprecation_rolling.policies.size.size = 1GB
-appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy
-appender.deprecation_rolling.strategy.max = 4
-
-logger.deprecation.name = org.elasticsearch.deprecation
-logger.deprecation.level = warn
-logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling
-logger.deprecation.additivity = false
-
-appender.index_search_slowlog_rolling.type = RollingFile
-appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs}_index_search_slowlog.log
-{% else %}
-appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog.log
-{% endif %}
-appender.index_search_slowlog_rolling.layout.type = PatternLayout
-appender.index_search_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs}_index_search_slowlog-%d{yyyy-MM-dd}.log
-{% else %}
-appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_search_slowlog-%d{yyyy-MM-dd}.log
-{% endif %}
-appender.index_search_slowlog_rolling.policies.type = Policies
-appender.index_search_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_search_slowlog_rolling.policies.time.interval = 1
-appender.index_search_slowlog_rolling.policies.time.modulate = true
-
-logger.index_search_slowlog_rolling.name = index.search.slowlog
-logger.index_search_slowlog_rolling.level = trace
-logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling
-logger.index_search_slowlog_rolling.additivity = false
-
-appender.index_indexing_slowlog_rolling.type = RollingFile
-appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs}_index_indexing_slowlog.log
-{% else %}
-appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog.log
-{% endif %}
-appender.index_indexing_slowlog_rolling.layout.type = PatternLayout
-appender.index_indexing_slowlog_rolling.layout.pattern = [%d{ISO8601}][%-5p][%-25c] %marker%.-10000m%n
-{% if (es_version | version_compare('6.0.0', '<')) %}
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-{% else %}
-appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_index_indexing_slowlog-%d{yyyy-MM-dd}.log
-{% endif %}
-appender.index_indexing_slowlog_rolling.policies.type = Policies
-appender.index_indexing_slowlog_rolling.policies.time.type = TimeBasedTriggeringPolicy
-appender.index_indexing_slowlog_rolling.policies.time.interval = 1
-appender.index_indexing_slowlog_rolling.policies.time.modulate = true
-
-logger.index_indexing_slowlog.name = index.indexing.slowlog.index
-logger.index_indexing_slowlog.level = trace
-logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
-logger.index_indexing_slowlog.additivity = false
diff --git a/ansible/roles/es6/templates/security/role_mapping.yml.j2 b/ansible/roles/es6/templates/security/role_mapping.yml.j2
deleted file mode 100644
index 2584375226..0000000000
--- a/ansible/roles/es6/templates/security/role_mapping.yml.j2
+++ /dev/null
@@ -1 +0,0 @@
-{{ es_role_mapping | to_nice_yaml }}
\ No newline at end of file
diff --git a/ansible/roles/es6/templates/security/roles.yml.j2 b/ansible/roles/es6/templates/security/roles.yml.j2
deleted file mode 100644
index 9f211f2b07..0000000000
--- a/ansible/roles/es6/templates/security/roles.yml.j2
+++ /dev/null
@@ -1 +0,0 @@
-{{ es_roles.file | to_nice_yaml }}
\ No newline at end of file
diff --git a/ansible/roles/es6/templates/security/users_roles.j2 b/ansible/roles/es6/templates/security/users_roles.j2
deleted file mode 100644
index 1c0acfa1d7..0000000000
--- a/ansible/roles/es6/templates/security/users_roles.j2
+++ /dev/null
@@ -1 +0,0 @@
-{{users_roles | join("\n") }}
\ No newline at end of file
diff --git a/ansible/roles/es6/templates/systemd/elasticsearch.j2 b/ansible/roles/es6/templates/systemd/elasticsearch.j2
deleted file mode 100644
index 8bd5545c02..0000000000
--- a/ansible/roles/es6/templates/systemd/elasticsearch.j2
+++ /dev/null
@@ -1,76 +0,0 @@
-[Unit]
-Description=Elasticsearch-{{es_instance_name}}
-Documentation=http://www.elastic.co
-Wants=network-online.target
-After=network-online.target
-
-[Service]
-Environment=ES_HOME={{es_home}}
-Environment=CONF_DIR={{conf_dir}}
-Environment=ES_PATH_CONF={{conf_dir}}
-Environment=DATA_DIR={{ data_dirs | array_to_str }}
-Environment=LOG_DIR={{log_dir}}
-Environment=PID_DIR={{pid_dir}}
-EnvironmentFile=-{{instance_default_file}}
-
-WorkingDirectory={{es_home}}
-
-User={{es_user}}
-Group={{es_group}}
-
-{% if (es_version | version_compare('6.0.0', '<')) %}
-ExecStartPre=/usr/share/elasticsearch/bin/elasticsearch-systemd-pre-exec
-{% endif %}
-
-ExecStart={{es_home}}/bin/elasticsearch \
-                                    -p ${PID_DIR}/elasticsearch.pid \
-{% if (es_version | version_compare('6.0.0', '<')) %}
-                                    -Edefault.path.logs=${LOG_DIR} \
-                                    -Edefault.path.data=${DATA_DIR} \
-                                    -Edefault.path.conf=${CONF_DIR} \
-{% endif %}
-                                    --quiet
-
-
-# StandardOutput is configured to redirect to journalctl since
-# some error messages may be logged in standard output before
-# elasticsearch logging system is initialized. Elasticsearch
-# stores its logs in /var/log/elasticsearch and does not use
-# journalctl by default. If you also want to enable journalctl
-# logging, you can simply remove the "quiet" option from ExecStart.
-StandardOutput=journal
-StandardError=inherit
-
-# Specifies the maximum file descriptor number that can be opened by this process
-{% if es_max_open_files is defined %}
-LimitNOFILE={{es_max_open_files}}
-{% endif %}
-
-# Specifies the maximum number of bytes of memory that may be locked into RAM
-# Set to "infinity" if you use the 'bootstrap.memory_lock: true' option
-# in elasticsearch.yml and 'MAX_LOCKED_MEMORY=unlimited' in {{instance_default_file}}
-{% if m_lock_enabled %}
-LimitMEMLOCK=infinity
-{% endif %}
-
-# Specifies the maximum number of threads that can be started.  Elasticsearch requires a
-# minimum of 2048.
-LimitNPROC={{ es_max_threads }}
-
-# Disable timeout logic and wait until process is stopped
-TimeoutStopSec=0
-
-# SIGTERM signal is used to stop the Java process
-KillSignal=SIGTERM
-
-# Send the signal only to the JVM rather than its control group
-KillMode=process
-
-# Java process is never killed
-SendSIGKILL=no
-
-# When a JVM receives a SIGTERM signal it exits with code 143
-SuccessExitStatus=143
-
-[Install]
-WantedBy=multi-user.target
diff --git a/ansible/roles/es6/vars/Debian.yml b/ansible/roles/es6/vars/Debian.yml
deleted file mode 100644
index 071736ef70..0000000000
--- a/ansible/roles/es6/vars/Debian.yml
+++ /dev/null
@@ -1,4 +0,0 @@
----
-java: "{% if es_java is defined %}{{es_java}}{% else %}openjdk-8-jre-headless{% endif %}"
-default_file: "/etc/default/elasticsearch"
-es_home: "/usr/share/elasticsearch"
diff --git a/ansible/roles/es6/vars/RedHat.yml b/ansible/roles/es6/vars/RedHat.yml
deleted file mode 100644
index b0aa42b2bb..0000000000
--- a/ansible/roles/es6/vars/RedHat.yml
+++ /dev/null
@@ -1,4 +0,0 @@
----
-java: "{{ es_java | default('java-1.8.0-openjdk.x86_64') }}"
-default_file: "/etc/sysconfig/elasticsearch"
-es_home: "/usr/share/elasticsearch"
\ No newline at end of file
diff --git a/ansible/roles/es6/vars/main.yml b/ansible/roles/es6/vars/main.yml
deleted file mode 100644
index b3bd126711..0000000000
--- a/ansible/roles/es6/vars/main.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-es_package_url: "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch"
-es_conf_dir: "/etc/elasticsearch"
-sysd_script: "/usr/lib/systemd/system/elasticsearch.service"
-init_script: "/etc/init.d/elasticsearch"
-#add supported features here
-supported_xpack_features: ["alerting","monitoring","graph","security"]
-reserved_xpack_users: ["elastic","kibana","logstash_system"]
\ No newline at end of file
diff --git a/ansible/roles/gcp-cloud-storage/defaults/main.yml b/ansible/roles/gcp-cloud-storage/defaults/main.yml
new file mode 100644
index 0000000000..b0fd847b26
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/defaults/main.yml
@@ -0,0 +1,49 @@
+# GCP bucket name
+# Example -
+# bucket_name: "sunbird-dev-public"
+gcp_bucket_name: ""
+
+# The service account key file
+# Example -
+# gcp_storage_key_file: "/tmp/gcp.json"
+gcp_storage_key_file: ""
+
+# Folder name in GCP bucket
+# Example -
+# gcp_path: "my-destination-folder"
+gcp_path: ""
+
+# The delete pattern to delete files and folder
+# Example -
+# file_delete_pattern: "my-drectory/*"
+# file_delete_pattern: "my-drectory/another-directory/*"
+# file_delete_pattern: "*"
+file_delete_pattern: ""
+
+# The path to local file which has to be uploaded to gcloud storage
+# The local path to store the file after downloading from gcloud storage
+# Example -
+# local_file_or_folder_path: "/workspace/my-folder/myfile.json"
+# local_file_or_folder_path: "/workspace/my-folder"
+local_file_or_folder_path: ""
+
+# The name of the file in gcloud storage after uploading from local path
+# The name of the file in gcloud storage that has to be downloaded
+# Example -
+# dest_file_name: "/myfile-blob.json"
+dest_file_name: ""
+
+
+# The folder path in gcloud storage to upload the files starting from the root of the bucket
+# This path should start with / if we provide a value for this variable since we are going to append this path as below
+# {{ bucket_name }}{{ gcp_path }}
+# The above translates to "my-bucket/my-folder-path"
+# Example -
+# dest_folder_path: "/my-folder/json-files-folder"
+# This variable can also be empty as shown below, which means we will upload directly at the root path of the bucket
+dest_folder_path: ""
+
+# The local folder path which has to be uploaded to gcloud storage
+# Example -
+# local_source_folder: "/workspace/my-folder/json-files-folder"
+local_source_folder: ""
diff --git a/ansible/roles/gcp-cloud-storage/tasks/delete-batch.yml b/ansible/roles/gcp-cloud-storage/tasks/delete-batch.yml
new file mode 100644
index 0000000000..17fe952b16
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/delete-batch.yml
@@ -0,0 +1,11 @@
+---
+- name: Authenticate to gcloud
+  include_tasks: gcloud-auth.yml
+
+- name: Delete folder recursively in gcp storage
+  shell: gsutil rm -r "gs://{{ gcp_bucket_name }}/{{ file_delete_pattern }}"
+  async: 3600
+  poll: 10
+
+- name: Revoke gcloud access
+  include_tasks: gcloud-revoke.yml
diff --git a/ansible/roles/gcp-cloud-storage/tasks/download.yml b/ansible/roles/gcp-cloud-storage/tasks/download.yml
new file mode 100644
index 0000000000..73bf76bb04
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/download.yml
@@ -0,0 +1,11 @@
+---
+- name: Authenticate to gcloud
+  include_tasks: gcloud-auth.yml
+
+- name: Download from gcloud storage
+  shell: gsutil cp "gs://{{ gcp_bucket_name }}/{{ gcp_path }}" "{{ local_file_or_folder_path }}"
+  async: 3600
+  poll: 10
+
+- name: Revoke gcloud access
+  include_tasks: gcloud-revoke.yml
diff --git a/ansible/roles/gcp-cloud-storage/tasks/gcloud-auth.yml b/ansible/roles/gcp-cloud-storage/tasks/gcloud-auth.yml
new file mode 100644
index 0000000000..a480bdc275
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/gcloud-auth.yml
@@ -0,0 +1,14 @@
+---
+- name: create tmp gcp service key file
+  tempfile:
+    state:  file
+    suffix: gcp
+  register: config_key
+
+- name: Copy service account key file
+  copy: 
+    content: "{{ gcp_storage_key_file }}" 
+    dest: "{{ config_key.path }}"
+
+- name: Configure gcloud service account
+  shell: gcloud auth activate-service-account "{{ gcp_storage_service_account_name }}" --key-file="{{ config_key.path }}"
diff --git a/ansible/roles/gcp-cloud-storage/tasks/gcloud-revoke.yml b/ansible/roles/gcp-cloud-storage/tasks/gcloud-revoke.yml
new file mode 100644
index 0000000000..8c26cd0ef0
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/gcloud-revoke.yml
@@ -0,0 +1,8 @@
+- name: Revoke gcloud service account access
+  shell: gcloud auth revoke "{{ gcp_storage_service_account_name }}"
+      
+- name: Remove key file
+  file:
+    path: "{{ config_key.path }}"
+    state: absent
+  when: config_key.path is defined
diff --git a/ansible/roles/gcp-cloud-storage/tasks/main.yml b/ansible/roles/gcp-cloud-storage/tasks/main.yml
new file mode 100644
index 0000000000..aa41c090ed
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/main.yml
@@ -0,0 +1,20 @@
+---
+- name: upload file to gcloud storage
+  include: upload.yml
+  tags:
+    - file-upload
+
+- name: upload batch of files to gcloud storage
+  include: upload-batch.yml
+  tags:
+    - upload-batch
+
+- name: delete batch of files from gcloud storage
+  include: delete-batch.yml
+  tags:
+    - delete-batch
+
+- name: download a file from gcloud storage
+  include: download.yml
+  tags:
+    - file-download
\ No newline at end of file
diff --git a/ansible/roles/gcp-cloud-storage/tasks/upload-batch.yml b/ansible/roles/gcp-cloud-storage/tasks/upload-batch.yml
new file mode 100644
index 0000000000..dc103969aa
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/upload-batch.yml
@@ -0,0 +1,11 @@
+---
+- name: Authenticate to gcloud
+  include_tasks: gcloud-auth.yml
+
+- name: Upload files from a local directory gcp storage
+  shell: gsutil -m cp -r "{{ local_file_or_folder_path }}" "gs://{{ gcp_bucket_name }}/{{ gcp_path}}"
+  async: 3600
+  poll: 10
+
+- name: Revoke gcloud access
+  include_tasks: gcloud-revoke.yml
diff --git a/ansible/roles/gcp-cloud-storage/tasks/upload.yml b/ansible/roles/gcp-cloud-storage/tasks/upload.yml
new file mode 100644
index 0000000000..de766a94c7
--- /dev/null
+++ b/ansible/roles/gcp-cloud-storage/tasks/upload.yml
@@ -0,0 +1,11 @@
+---
+- name: Authenticate to gcloud
+  include_tasks: gcloud-auth.yml
+
+- name: Upload to gcloud storage
+  shell: gsutil cp "{{ local_file_or_folder_path }}" "gs://{{ gcp_bucket_name }}/{{ gcp_path }}"
+  async: 3600
+  poll: 10
+
+- name: Revoke gcloud access
+  include_tasks: gcloud-revoke.yml
diff --git a/ansible/roles/graphite-exporter/README.md b/ansible/roles/graphite-exporter/README.md
new file mode 100644
index 0000000000..7dc55d1b67
--- /dev/null
+++ b/ansible/roles/graphite-exporter/README.md
@@ -0,0 +1,16 @@
+## Graphite exporter as systemd
+
+This role will install graphite exporter as systemd process in VM for applications which emit metrics in graphite format to injest the metrics in to prometheus.
+
+### Applications
+
+1. Druid:
+    1. Enable druid [metrics](https://github.com/apache/druid/blob/master/docs/operations/metrics.md) in `/data/druid/whitelist` 
+       ref: 
+    2. Add below config in `/data/druid/conf/druid/_common/common.runtime.properties`
+       ```
+       druid.emitter.graphite.port=9109
+       druid.emitter.graphite.hostname=localhost
+       druid.emitter.graphite.protocol=plaintext
+       ```
+    3. Add scrape config in prometheus to scrape `this_machine_ip:9108` port
diff --git a/ansible/roles/graphite-exporter/defaults/main.yml b/ansible/roles/graphite-exporter/defaults/main.yml
new file mode 100644
index 0000000000..6de223f0de
--- /dev/null
+++ b/ansible/roles/graphite-exporter/defaults/main.yml
@@ -0,0 +1,12 @@
+---
+
+graphite_exporter_version: 0.7.1
+
+graphite_exporter_binary_url: 'https://github.com/prometheus/graphite_exporter/releases/download/v{{graphite_exporter_version}}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64.tar.gz'
+
+graphite_exporter_user: graphite-exp
+graphite_exporter_group: graphite-exp
+
+graphite_exporter_root_dir: /opt/graphite_exporter
+graphite_exporter_dist_dir: "{{ graphite_exporter_root_dir }}/dist"
+graphite_exporter_config_dir: "{{ graphite_exporter_root_dir }}/config"
diff --git a/ansible/roles/graphite-exporter/handlers/main.yml b/ansible/roles/graphite-exporter/handlers/main.yml
new file mode 100644
index 0000000000..cee21aab37
--- /dev/null
+++ b/ansible/roles/graphite-exporter/handlers/main.yml
@@ -0,0 +1,15 @@
+---
+
+- name: reenable graphite exporter service
+  systemd:
+    daemon_reload: yes
+    enabled: yes
+    name: prometheus-graphite-exporter
+
+- name: reinit graphite exporter
+  command: initctl reload-configuration
+
+- name: restart graphite exporter
+  service:
+    name: prometheus-graphite-exporter
+    state: restarted
diff --git a/ansible/roles/graphite-exporter/tasks/main.yml b/ansible/roles/graphite-exporter/tasks/main.yml
new file mode 100644
index 0000000000..7edfe23dde
--- /dev/null
+++ b/ansible/roles/graphite-exporter/tasks/main.yml
@@ -0,0 +1,71 @@
+---
+
+- name: 'create druid exporter group'
+  group:
+    name: '{{ graphite_exporter_group }}'
+    system: yes
+    state: present
+
+- name: 'create graphite exporter user'
+  user:
+    name: '{{ graphite_exporter_user }}'
+    system: yes
+    shell: '/sbin/nologin'
+    group: '{{ graphite_exporter_group }}'
+    createhome: no
+
+- name: 'create graphite exporter directories'
+  file:
+    path: '{{ item }}'
+    state: directory
+    owner: '{{ graphite_exporter_user }}'
+    group: '{{ graphite_exporter_group }}'
+    mode: 0755
+  with_items:
+    - '{{ graphite_exporter_root_dir }}'
+    - '{{ graphite_exporter_dist_dir }}'
+    - '{{ graphite_exporter_config_dir }}'
+
+- name: download and unarchive graphite exporter binary
+  unarchive:
+    src: '{{ graphite_exporter_binary_url }}'
+    dest: '{{ graphite_exporter_dist_dir }}/'
+    mode: 0755
+    remote_src: yes
+
+- name: 'Copy graphite exporter config'
+  template:
+    src: config.j2
+    dest: '{{ graphite_exporter_config_dir }}/graphite_mapping.conf'
+  notify:
+    - restart graphite exporter
+
+- name: 'update group and owner for files'
+  file:
+    path: '{{ item }}'
+    state: file
+    owner: '{{ graphite_exporter_user }}'
+    group: '{{ graphite_exporter_group }}'
+  with_items:
+    - '{{ graphite_exporter_dist_dir }}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64/graphite_exporter'
+    - '{{ graphite_exporter_config_dir }}/graphite_mapping.conf'
+
+- name: 'create systemd service unit'
+  template:
+    src: prometheus-graphite-exporter.service.j2
+    dest: /etc/systemd/system/prometheus-graphite-exporter.service
+    owner: root
+    group: root
+    mode: 0644
+  when: service_mgr | default(ansible_service_mgr) == 'systemd'
+  notify:
+    - reenable graphite exporter service
+    - restart graphite exporter
+
+- meta: flush_handlers
+
+- name: 'ensure prometheus graphite exporter service is enabled and started'
+  service:
+    name: prometheus-graphite-exporter
+    state: started
+    enabled: yes
diff --git a/ansible/roles/graphite-exporter/templates/config.j2 b/ansible/roles/graphite-exporter/templates/config.j2
new file mode 100644
index 0000000000..36069ed49b
--- /dev/null
+++ b/ansible/roles/graphite-exporter/templates/config.j2
@@ -0,0 +1,21 @@
+# This config is based on observed metrics from druid
+# For this to work in druid /data/druid/conf/druid/_common/common.runtime.properties
+
+mappings:
+- match: druid.metric.*.*.*
+  name: druid_$3
+  labels:
+    role: $1
+    ip: $2
+- match: druid.metric.*.*.*.*
+  name: druid_$4
+  labels:
+    role: $1
+    ip: $2
+    job: $3
+- match: druid.metric.*.*.*.*.*
+  name: druid_${4}_${5}
+  labels:
+    role: $1
+    ip: $2
+    job: $3
diff --git a/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2 b/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2
new file mode 100644
index 0000000000..28435d0289
--- /dev/null
+++ b/ansible/roles/graphite-exporter/templates/prometheus-graphite-exporter.service.j2
@@ -0,0 +1,15 @@
+[Unit]
+Description=Prometheus graphite Exporter
+After=network.target
+
+[Service]
+Type=simple
+User={{ graphite_exporter_user }}
+Group={{ graphite_exporter_group }}
+ExecStart={{ graphite_exporter_dist_dir }}/graphite_exporter-{{graphite_exporter_version}}.linux-amd64/graphite_exporter --graphite.mapping-config="{{ graphite_exporter_config_dir }}/graphite_mapping.conf" --graphite.listen-address=":9109" --web.listen-address=":9108" --graphite.sample-expiry=1h --graphite.cache-size=10000
+
+SyslogIdentifier=prometheus_graphite_exporter
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/influxdb_backup/defaults/main.yml b/ansible/roles/influxdb_backup/defaults/main.yml
index c92635b613..4104902eb6 100644
--- a/ansible/roles/influxdb_backup/defaults/main.yml
+++ b/ansible/roles/influxdb_backup/defaults/main.yml
@@ -2,4 +2,6 @@ influxdb_backup_dir: /tmp/influxdb_backup
 influxdb_backup_databases: ["monitoring_events"]
 influxdb_backup_file_prefix: influxdb_backup
 influxdb_backup_file_name: "{{ influxdb_backup_file_prefix }}-{{ ansible_date_time.date }}-{{ ansible_date_time.hour }}-{{ ansible_date_time.minute }}-{{ ansible_date_time.second }}"
-azure_influxdb_backup_container: influxdb-backup
\ No newline at end of file
+
+cloud_storage_influxdbbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_influxdbbackup_foldername: influxdb-backup
diff --git a/ansible/roles/influxdb_backup/tasks/main.yml b/ansible/roles/influxdb_backup/tasks/main.yml
index b51a34291d..2ddf64e950 100644
--- a/ansible/roles/influxdb_backup/tasks/main.yml
+++ b/ansible/roles/influxdb_backup/tasks/main.yml
@@ -5,16 +5,48 @@
 - name: Backup the database
   command: "influxd backup -portable -db {{ item }} {{ influxdb_backup_dir }}"
   with_items: "{{ influxdb_backup_databases }}"
-
-- name: Ensure azure blob storage container exists
-  command: az storage container create  --name "{{azure_influxdb_backup_container}}" --public-access blob
-
+  
 - name: zip the backup dir
   archive:
     path: "{{ influxdb_backup_dir }}"
     dest: "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}.zip"
     format: zip
+    
+- name: upload file to azure storage using azcopy
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: upload-using-azcopy.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_influxdbbackup_foldername }}"
+    container_public_access: "off"
+    blob_container_folder_path: ""
+    local_file_or_folder_path: "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}.zip"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_sas_token: "{{ azure_management_storage_account_sas }}"
+  when: cloud_service_provider == "azure"
+
+- name: upload file to aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_management_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}.zip"
+    s3_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_backup_file_name }}.zip"
+  when: cloud_service_provider == "aws"
+
+- name: upload file to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_management_bucketname }}"
+    gcp_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_backup_file_name }}.zip"
+    local_file_or_folder_path: "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}.zip"
+  when: cloud_service_provider == "gcloud"
 
-- name: Upload the backup to azure blob storage
-  #command: az storage blob upload --name "{{ influxdb_backup_file_name }}".zip --file "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}".zip --container-name {{ azure_influxdb_backup_container }}
-  command: az storage blob upload -c "{{ azure_influxdb_backup_container }}" --name "{{ influxdb_backup_file_name }}".zip -f "{{ influxdb_backup_dir }}/{{ influxdb_backup_file_name }}".zip
diff --git a/ansible/roles/influxdb_restore/defaults/main.yml b/ansible/roles/influxdb_restore/defaults/main.yml
index ee100559da..c865e9ab49 100644
--- a/ansible/roles/influxdb_restore/defaults/main.yml
+++ b/ansible/roles/influxdb_restore/defaults/main.yml
@@ -1 +1,2 @@
-influxdb_backup_azure_container_name: influxdb-backup 
+cloud_storage_influxdbbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_influxdbbackup_foldername: influxdb-backup
diff --git a/ansible/roles/influxdb_restore/meta/main.yml b/ansible/roles/influxdb_restore/meta/main.yml
deleted file mode 100644
index a124d4f7cb..0000000000
--- a/ansible/roles/influxdb_restore/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
diff --git a/ansible/roles/influxdb_restore/tasks/main.yml b/ansible/roles/influxdb_restore/tasks/main.yml
index 0681467405..7a2317caa0 100644
--- a/ansible/roles/influxdb_restore/tasks/main.yml
+++ b/ansible/roles/influxdb_restore/tasks/main.yml
@@ -6,10 +6,43 @@
     update_cache:  yes
     state: present
 
-- name: Download backup file
-  shell: "az storage blob download --container-name {{ influxdb_backup_azure_container_name }} --file {{ influxdb_restore_file_name }} --name {{ influxdb_restore_file_name }} --account-name {{sunbird_management_storage_account_name}} --account-key {{sunbird_management_storage_account_key}}"
-  args:
-    chdir: /tmp/
+- name: download a file from azure storage
+  become: true
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_influxdbbackup_foldername }}"
+    blob_file_name: "{{ influxdb_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ influxdb_restore_file_name  }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+
+- name: download a file from aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_influxdbbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ postgres_restore_dir }}/{{ influxdb_restore_file_name }}"
+    s3_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_restore_file_name }}"
+  when: cloud_service_provider == "aws"
+
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_influxdbbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_influxdbbackup_foldername }}/{{ influxdb_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ influxdb_restore_file_name }}"
+  when: cloud_service_provider == "gcloud"
 
 - name: Unarchieve backup file
   unarchive:
diff --git a/ansible/roles/jdk11/tasks/main.yml b/ansible/roles/jdk11/tasks/main.yml
new file mode 100644
index 0000000000..46b491df9f
--- /dev/null
+++ b/ansible/roles/jdk11/tasks/main.yml
@@ -0,0 +1,24 @@
+---
+# You can use JAVA_HOME=/opt/jdk-11 in your applications / exports if you want to use JDK11
+
+- name: Uncompressing and copying to system path
+  unarchive:
+    src: https://download.java.net/java/GA/jdk11/9/GPL/openjdk-11.0.2_linux-x64_bin.tar.gz
+    dest: /opt/
+    remote_src: yes
+    keep_newer: yes
+  register: tar
+
+- name: Updating java jdk-11 to system path
+  shell: |
+    update-alternatives --install /usr/bin/java java /opt/jdk-11.0.2/bin/java 9998
+    update-alternatives --install /usr/bin/javac javac /opt/jdk-11.0.2/bin/java 9998
+    sudo update-alternatives --set java /opt/jdk-11.0.2/bin/java
+- name: Checking java version
+  shell: java -version
+  register: java
+  changed_when: false
+
+- name: Java version
+  debug:
+    var: java.stderr
\ No newline at end of file
diff --git a/ansible/roles/kibana/.travis.yml b/ansible/roles/kibana/.travis.yml
deleted file mode 100644
index 0767a46dba..0000000000
--- a/ansible/roles/kibana/.travis.yml
+++ /dev/null
@@ -1,65 +0,0 @@
----
-sudo: required
-
-env:
-  global:
-    # https://github.com/travis-ci/travis-ci/issues/6461#issuecomment-239577306
-    DOCKER_VERSION: "1.9.1-0~trusty"
-  matrix:
-    - distro: centos7
-      init: /usr/lib/systemd/systemd
-      run_opts: "--privileged --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro"
-    - distro: centos6
-      init: /sbin/init
-      run_opts: ""
-    - distro: ubuntu1604
-      init: /lib/systemd/systemd
-      run_opts: "--privileged --volume=/sys/fs/cgroup:/sys/fs/cgroup:ro"
-    - distro: ubuntu1404
-      init: /sbin/init
-      run_opts: ""
-    - distro: ubuntu1204
-      init: /sbin/init
-      run_opts: ""
-
-services:
-  - docker
-
-before_install:
-  # Downgrade to specific version of Docker engine.
-  - sudo apt-get update
-  - sudo apt-get remove docker-engine -yq
-  - sudo apt-get install docker-engine=$DOCKER_VERSION -yq --no-install-suggests --no-install-recommends --force-yes -o Dpkg::Options::="--force-confnew"
-
-  # Pull container.
-  - 'sudo docker pull geerlingguy/docker-${distro}-ansible:latest'
-
-script:
-  - container_id=$(mktemp)
-  # Run container in detached state.
-  - 'sudo docker run --detach --volume="${PWD}":/etc/ansible/roles/role_under_test:ro ${run_opts} geerlingguy/docker-${distro}-ansible:latest "${init}" > "${container_id}"'
-
-  # Ansible syntax check.
-  - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml --syntax-check'
-
-  # Test role.
-  - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml'
-
-  # Test role idempotence.
-  - idempotence=$(mktemp)
-  - sudo docker exec "$(cat ${container_id})" ansible-playbook /etc/ansible/roles/role_under_test/tests/test.yml | tee -a ${idempotence}
-  - >
-    tail ${idempotence}
-    | grep -q 'changed=0.*failed=0'
-    && (echo 'Idempotence test: pass' && exit 0)
-    || (echo 'Idempotence test: fail' && exit 1)
-
-  # Test whether Kibana is running correctly.
-  - 'sudo docker exec --tty "$(cat ${container_id})" env TERM=xterm curl http://localhost:5601/'
-
-after_success:
-  # Clean up.
-  - 'sudo docker stop "$(cat ${container_id})"'
-
-notifications:
-  webhooks: https://galaxy.ansible.com/api/v1/notifications/
diff --git a/ansible/roles/kibana/README.md b/ansible/roles/kibana/README.md
deleted file mode 100644
index ac0877ba49..0000000000
--- a/ansible/roles/kibana/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-# Ansible Role: Kibana
-
-[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-kibana.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-kibana)
-
-An Ansible Role that installs Kibana on RedHat/CentOS or Debian/Ubuntu.
-
-## Requirements
-
-None.
-
-## Role Variables
-
-Available variables are listed below, along with default values (see `defaults/main.yml`):
-
-    kibana_version: "4.6"
-
-The version of kibana to install (major and minor only).
-
-    kibana_server_port: 5601
-    kibana_server_host: "0.0.0.0"
-
-The FQDN or IP address and port Kibana should use.
-
-    kibana_elasticsearch_url: "http://localhost:9200"
-
-The URL (including port) over which Kibana will connect to Elasticsearch.
-
-## Dependencies
-
-None.
-
-## Example Playbook
-
-    - hosts: kibana
-      roles:
-        - geerlingguy.kibana
-
-## License
-
-MIT / BSD
-
-## Author Information
-
-This role was created in 2014 by [Jeff Geerling](http://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).
diff --git a/ansible/roles/kibana/defaults/main.yml b/ansible/roles/kibana/defaults/main.yml
deleted file mode 100644
index fb22975eb3..0000000000
--- a/ansible/roles/kibana/defaults/main.yml
+++ /dev/null
@@ -1,7 +0,0 @@
----
-kibana_version: "6.2.3"
-
-kibana_base_path: "/pipeline-dashboard"
-kibana_server_port: 5601
-kibana_server_host: "0.0.0.0"
-kibana_elasticsearch_url: "http://localhost:9200"
diff --git a/ansible/roles/kibana/handlers/main.yml b/ansible/roles/kibana/handlers/main.yml
deleted file mode 100644
index 55ea3d3c1a..0000000000
--- a/ansible/roles/kibana/handlers/main.yml
+++ /dev/null
@@ -1,3 +0,0 @@
----
-- name: restart kibana
-  service: name=kibana state=restarted
diff --git a/ansible/roles/kibana/meta/.galaxy_install_info b/ansible/roles/kibana/meta/.galaxy_install_info
deleted file mode 100644
index 0983f9522c..0000000000
--- a/ansible/roles/kibana/meta/.galaxy_install_info
+++ /dev/null
@@ -1 +0,0 @@
-{install_date: 'Wed Apr 25 07:28:43 2018', version: 2.0.1}
diff --git a/ansible/roles/kibana/meta/main.yml b/ansible/roles/kibana/meta/main.yml
deleted file mode 100644
index dbbf992b98..0000000000
--- a/ansible/roles/kibana/meta/main.yml
+++ /dev/null
@@ -1,26 +0,0 @@
----
-dependencies: []
-
-galaxy_info:
-  author: geerlingguy
-  description: Kibana for Linux.
-  company: "Midwestern Mac, LLC"
-  license: "license (BSD, MIT)"
-  min_ansible_version: 2.0
-  platforms:
-  - name: EL
-    versions:
-    - 6
-    - 7
-  - name: Debian
-    versions:
-    - jessie
-  - name: Ubuntu
-    versions:
-    - precise
-    - trusty
-    - xenial
-  galaxy_tags:
-    - web
-    - system
-    - monitoring
diff --git a/ansible/roles/kibana/tasks/main.yml b/ansible/roles/kibana/tasks/main.yml
deleted file mode 100644
index 51245f98c5..0000000000
--- a/ansible/roles/kibana/tasks/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
----
-- include: setup-RedHat.yml
-  when: ansible_os_family == 'RedHat'
-
-- include: setup-Debian.yml
-  when: ansible_os_family == 'Debian'
-
-- name: Ensure Kibana is started and enabled at boot.
-  service:
-    name: kibana
-    state: started
-    enabled: yes
-
-- name: Copy Kibana configuration.
-  template:
-    src: kibana.yml.j2
-    dest: "/etc/kibana/kibana.yml"
-    owner: root
-    group: root
-    mode: 0644
-  notify: restart kibana
diff --git a/ansible/roles/kibana/tasks/setup-Debian.yml b/ansible/roles/kibana/tasks/setup-Debian.yml
deleted file mode 100644
index e0cc720da6..0000000000
--- a/ansible/roles/kibana/tasks/setup-Debian.yml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-- name: Ensure depdency is installed (Ubuntu).
-  apt: name=apt-transport-https state=present
-
-- name: Download Kibana
-  get_url:
-    url: "https://artifacts.elastic.co/downloads/kibana/kibana-{{ kibana_version }}-amd64.deb"
-    dest: /tmp/kibana.deb
-
-- name: Install Kibana
-  apt: deb="/tmp/kibana.deb"
-
diff --git a/ansible/roles/kibana/tasks/setup-RedHat.yml b/ansible/roles/kibana/tasks/setup-RedHat.yml
deleted file mode 100644
index bdb463ea48..0000000000
--- a/ansible/roles/kibana/tasks/setup-RedHat.yml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-- name: Add Elasticsearch GPG key.
-  rpm_key:
-    key: https://packages.elastic.co/GPG-KEY-elasticsearch
-    state: present
-
-- name: Add Kibana repository.
-  template:
-    src: kibana.repo.j2
-    dest: /etc/yum.repos.d/kibana.repo
-    mode: 0644
diff --git a/ansible/roles/kibana/templates/kibana.repo.j2 b/ansible/roles/kibana/templates/kibana.repo.j2
deleted file mode 100644
index 51bfec336a..0000000000
--- a/ansible/roles/kibana/templates/kibana.repo.j2
+++ /dev/null
@@ -1,6 +0,0 @@
-[kibana-{{ kibana_version }}]
-name=Kibana repository for {{ kibana_version }}.x packages
-baseurl=https://packages.elastic.co/kibana/{{ kibana_version }}/centos
-gpgcheck=1
-gpgkey=https://packages.elastic.co/GPG-KEY-elasticsearch
-enabled=1
diff --git a/ansible/roles/kibana/templates/kibana.yml.j2 b/ansible/roles/kibana/templates/kibana.yml.j2
deleted file mode 100644
index 639ca4e86f..0000000000
--- a/ansible/roles/kibana/templates/kibana.yml.j2
+++ /dev/null
@@ -1,83 +0,0 @@
-# Kibana is served by a back end server. This controls which port to use.
-server.port: {{ kibana_server_port }}
-
-# The host to bind the server to.
-server.host: {{ kibana_server_host }}
-
-# If you are running kibana behind a proxy, and want to mount it at a path,
-# specify that path here. The basePath can't end in a slash.
-server.basePath: {{ kibana_base_path }}
-
-# The maximum payload size in bytes on incoming server requests.
-# server.maxPayloadBytes: 1048576
-
-# The Elasticsearch instance to use for all your queries.
-elasticsearch.url: {{ kibana_elasticsearch_url }}
-
-# preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false,
-# then the host you use to connect to *this* Kibana instance will be sent.
-# elasticsearch.preserveHost: true
-
-# Kibana uses an index in Elasticsearch to store saved searches, visualizations
-# and dashboards. It will create a new index if it doesn't already exist.
-# kibana.index: ".kibana"
-
-# The default application to load.
-# kibana.defaultAppId: "discover"
-
-# If your Elasticsearch is protected with basic auth, these are the user credentials
-# used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana
-# users will still need to authenticate with Elasticsearch (which is proxied through
-# the Kibana server)
-# elasticsearch.username: "user"
-# elasticsearch.password: "pass"
-
-# SSL for outgoing requests from the Kibana Server to the browser (PEM formatted)
-# server.ssl.cert: /path/to/your/server.crt
-# server.ssl.key: /path/to/your/server.key
-
-# Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted)
-# elasticsearch.ssl.cert: /path/to/your/client.crt
-# elasticsearch.ssl.key: /path/to/your/client.key
-
-# If you need to provide a CA certificate for your Elasticsearch instance, put
-# the path of the pem file here.
-# elasticsearch.ssl.ca: /path/to/your/CA.pem
-
-# Set to false to have a complete disregard for the validity of the SSL
-# certificate.
-# elasticsearch.ssl.verify: true
-
-# Time in milliseconds to wait for elasticsearch to respond to pings, defaults to
-# request_timeout setting
-# elasticsearch.pingTimeout: 1500
-
-# Time in milliseconds to wait for responses from the back end or elasticsearch.
-# This must be > 0
-# elasticsearch.requestTimeout: 30000
-
-# Header names and values that are sent to Elasticsearch. Any custom headers cannot be overwritten
-# by client-side headers.
-# elasticsearch.customHeaders: {}
-
-# Time in milliseconds for Elasticsearch to wait for responses from shards.
-# Set to 0 to disable.
-# elasticsearch.shardTimeout: 0
-
-# Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying
-# elasticsearch.startupTimeout: 5000
-
-# Set the path to where you would like the process id file to be created.
-# pid.file: /var/run/kibana.pid
-
-# If you would like to send the log output to a file you can set the path below.
-# logging.dest: stdout
-
-# Set this to true to suppress all logging output.
-# logging.silent: false
-
-# Set this to true to suppress all logging output except for error messages.
-# logging.quiet: false
-
-# Set this to true to log all events, including system usage information and all requests.
-# logging.verbose: false
diff --git a/ansible/roles/kibana/tests/test.yml b/ansible/roles/kibana/tests/test.yml
deleted file mode 100644
index 991eb5bef8..0000000000
--- a/ansible/roles/kibana/tests/test.yml
+++ /dev/null
@@ -1,13 +0,0 @@
----
-- hosts: all
-
-  pre_tasks:
-    - name: Update apt cache.
-      apt: update_cache=yes
-      when: ansible_os_family == 'Debian'
-
-    - name: Install test dependencies.
-      package: name=curl state=present
-
-  roles:
-    - role_under_test
diff --git a/ansible/roles/monit/defaults/main.yml b/ansible/roles/monit/defaults/main.yml
index 4b2174f4f7..d1744ddddb 100644
--- a/ansible/roles/monit/defaults/main.yml
+++ b/ansible/roles/monit/defaults/main.yml
@@ -157,6 +157,70 @@ monit_learning-failed-backup:
   pid_file: /home/analytics/sbin/learning-failed-backup.pid
   retry: 3
 
+monit_ingestion-cluster-telemetry-backup:
+  app_name: ingestion-cluster-telemetry-backup
+  custom_start: /home/analytics/sbin/secor ingestion-cluster-telemetry-backup start
+  custom_stop: /home/analytics/sbin/secor ingestion-cluster-telemetry-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/ingestion-cluster-telemetry-backup.pid
+  retry: 3
+
+monit_assess-raw-events-backup:
+  app_name: assess-raw-events-backup
+  custom_start: /home/analytics/sbin/secor assess-raw-events-backup start
+  custom_stop: /home/analytics/sbin/secor assess-raw-events-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/assess-raw-events-backup.pid
+  retry: 3
+
+monit_telemetry-duplicate-backup:
+  app_name: telemetry-duplicate-backup
+  custom_start: /home/analytics/sbin/secor telemetry-duplicate-backup start
+  custom_stop: /home/analytics/sbin/secor telemetry-duplicate-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/telemetry-duplicate-backup.pid
+  retry: 3
+
+monit_telemetry-ingest-backup:
+  app_name: telemetry-ingest-backup
+  custom_start: /home/analytics/sbin/secor telemetry-ingest-backup start
+  custom_stop: /home/analytics/sbin/secor telemetry-ingest-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/telemetry-ingest-backup.pid
+  retry: 3
+
+monit_extractor-duplicate-backup:
+  app_name: extractor-duplicate-backup
+  custom_start: /home/analytics/sbin/secor extractor-duplicate-backup start
+  custom_stop: /home/analytics/sbin/secor extractor-duplicate-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/extractor-duplicate-backup.pid
+  retry: 3
+
+monit_content-consumption-events-backup:
+  app_name: content-consumption-events-backup
+  custom_start: /home/analytics/sbin/secor content-consumption-events-backup start
+  custom_stop: /home/analytics/sbin/secor content-consumption-events-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/content-consumption-events-backup.pid
+  retry: 3
+
+monit_derived-denorm-events-backup:
+  app_name: derived-denorm-events-backup
+  custom_start: /home/analytics/sbin/secor derived-denorm-events-backup start
+  custom_stop: /home/analytics/sbin/secor derived-denorm-events-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/derived-denorm-events-backup.pid
+  retry: 3
+
+monit_channel-summary-backup:
+  app_name: channel-summary-backup
+  custom_start: /home/analytics/sbin/secor channel-summary-backup start
+  custom_stop: /home/analytics/sbin/secor channel-summary-backup stop
+  user: analytics
+  pid_file: /home/analytics/sbin/channel-summary-backup.pid
+  retry: 3
+
 monit_cassandra-dp:
   app_name: "cassandra"
   custom_start: /bin/systemctl start cassandra
diff --git a/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml b/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml
deleted file mode 100644
index 7472ab4e27..0000000000
--- a/ansible/roles/postgres-azure-managed-service-restore/defaults/main.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-postgresql_restore_dir: /tmp/postgres-restore
-postgres_backup_azure_container_name: dp-postgresql-backup
-
-db:
-  name: ['analytics']
-  role: ['analytics']
-
-#these variables are passed as extra vars
-postgres_backup_filename:
-postgres_user: 
-postgres_password:
-postgres_hostname: 
diff --git a/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml b/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml
deleted file mode 100644
index 36fc39e949..0000000000
--- a/ansible/roles/postgres-azure-managed-service-restore/tasks/main.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-- name: install psycopg2
-  package:
-    name: python-psycopg2
-    state: present
-  
-- name: ensure restore dir exists
-  file: path="{{ postgresql_restore_dir }}" state=directory mode=0777
-
-- set_fact:
-    postgres_backup_filepath: "{{ postgresql_restore_dir }}/{{ postgres_backup_filename }}"
-
-- name: Download backup from azure
-  command: az storage blob download -c {{ postgres_backup_azure_container_name }} --name {{ postgres_backup_filename }} -f {{ postgres_backup_filepath }}
-  args:
-    chdir: "{{ postgres_restore_dir }}"
-  async: 100
-  poll: 10
-
-- name: unarchive artifact
-  unarchive: src={{ postgresql_restore_dir }}/{{ postgres_backup_filename }} dest={{ postgresql_restore_dir }}/ copy=no
-- name: Create db's
-  postgresql_db:
-    login_user: "{{ postgres_user }}"
-    login_password: "{{ postgres_password }}"
-    login_host: "{{ postgres_hostname }}"
-    name: "{{ item }}"
-    state: present
-  with_items: "{{ db.name }}"
-  async: 1000
-  poll: 10
-
-- name: Create role and grant access to db's
-  postgresql_user:
-    login_user: "{{ postgres_user }}"
-    login_password: "{{ postgres_password }}"
-    login_host: "{{ postgres_hostname }}"
-    db: "{{ item[0] }}"
-    name: "{{ item[1] }}"
-    priv: ALL
-    state: present
-    role_attr_flags: CREATEROLE
-  with_nested:
-    - "{{ db.name }}"
-    - "{{ db.role }}"
-  async: 1000
-  poll: 10
-
-- name: Restore db's
-  postgresql_db:
-    login_user: "{{ postgres_user }}"
-    login_password: "{{ postgres_password }}"
-    login_host: "{{ postgres_hostname }}"
-    name: "{{ item }}"
-    state: restore
-    target: "{{ item }}.sql"
-  args:
-    chdir: "{{ postgres_restore_dir }}"
-  with_items: "{{ db.name }}"
-
-- name: clean up restore dir
-  file: path="{{ postgresql_restore_dir }}" state=absent
-
diff --git a/ansible/roles/postgres-azure-managed-service/defaults/main.yml b/ansible/roles/postgres-azure-managed-service/defaults/main.yml
deleted file mode 100644
index e5d738fc6c..0000000000
--- a/ansible/roles/postgres-azure-managed-service/defaults/main.yml
+++ /dev/null
@@ -1,6 +0,0 @@
-postgresql_user: postgres
-postgresql_backup_dir: /tmp/dp-postgres
-postgresql_backup_azure_container_name: dp-postgresql-backup
-
-db_name:
-  db: ['analytics']
\ No newline at end of file
diff --git a/ansible/roles/postgres-azure-managed-service/meta/main.yml b/ansible/roles/postgres-azure-managed-service/meta/main.yml
deleted file mode 100644
index 5927f82724..0000000000
--- a/ansible/roles/postgres-azure-managed-service/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  #- azure-cli
\ No newline at end of file
diff --git a/ansible/roles/postgres-azure-managed-service/tasks/main.yml b/ansible/roles/postgres-azure-managed-service/tasks/main.yml
deleted file mode 100644
index ab415d7566..0000000000
--- a/ansible/roles/postgres-azure-managed-service/tasks/main.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-- name: install psycopg2
-  package:
-    name: python-psycopg2
-    state: present
-- name: ensure backup dir exists
-  file: path="{{ postgresql_backup_dir }}" state=directory mode=0777
-
-- set_fact:
-    postgresql_backup_gzip_file_name: "postgresql_backup_{{ lookup('pipe', 'date +%Z-%Y-%m-%d-%H-%M-%S') }}"
-
-- set_fact:
-    postgresql_backup_gzip_file_path: "{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}.zip"
-
-
-- name: Dump an existing database to a file
-  postgresql_db:
-    login_user: "{{ postgres.db_username }}"
-    login_password: "{{ postgres.db_password }}"
-    login_host: "{{ postgres.db_url }}"
-    name: "{{ item }}"
-    state: dump
-    target: "{{ postgresql_backup_dir }}/{{ item }}.sql"
-  with_items: "{{ db_name.db }}"
-  async: 3600
-  poll: 10
-
-- name: Create archive of backup directory
-  archive: path="{{ postgresql_backup_dir }}/*" dest="{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}.zip" format=zip
-  async: 500
-  poll: 10
-
-- name: Ensure azure blob storage container exists
-  command: az storage container create  --name {{ postgresql_backup_azure_container_name }}
-  ignore_errors: true
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-
-- name: Upload to azure blob storage
-  command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }}.zip --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }}
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-  async: 3600
-  poll: 10
-
-- name: clean up backup dir after upload
-  file: path="{{ postgresql_backup_dir }}" state=absent
diff --git a/ansible/roles/postgres/defaults/main.yml b/ansible/roles/postgres-db-update/defaults/main.yml
similarity index 100%
rename from ansible/roles/postgres/defaults/main.yml
rename to ansible/roles/postgres-db-update/defaults/main.yml
diff --git a/ansible/roles/postgres-db-update/tasks/main.yml b/ansible/roles/postgres-db-update/tasks/main.yml
new file mode 100644
index 0000000000..631299b253
--- /dev/null
+++ b/ansible/roles/postgres-db-update/tasks/main.yml
@@ -0,0 +1,167 @@
+- name: Install PostgreSQL client
+  apt: name={{ item }} update_cache=yes cache_valid_time=36000 state=installed
+  with_items:
+    - python-psycopg2
+    - postgresql-client
+  ignore_errors: yes
+
+- name: Ensure database is created
+  postgresql_db: name="{{ postgres.db_name }}" \
+                login_host="{{ postgres.db_url }}" \
+                port="{{ postgres.db_port }}" \
+                login_user="{{ postgres.db_admin_user }}" \
+                login_password="{{ postgres.db_admin_password }}" \
+                encoding='UTF-8' \
+                state=present
+  tags: createdb
+
+- name: Ensure database for superset is created
+  postgresql_db: name="superset" \
+                login_host="{{ postgres.db_url }}" \
+                port="{{ postgres.db_port }}" \
+                login_user="{{ postgres.db_admin_user }}" \
+                login_password="{{ postgres.db_admin_password }}" \
+                encoding='UTF-8' \
+                state=present
+  tags: createdb
+
+- name: Ensure user has access to the database
+  postgresql_user: name="{{ postgres.db_username }}" \
+                  password="{{ postgres.db_password }}" \
+                  no_password_changes=true \
+                  priv=ALL \
+                  state=present \
+                  login_host="{{ postgres.db_url }}" \
+                  port="{{ postgres.db_port }}" \
+                  login_user="{{ postgres.db_admin_user }}" \
+                  login_password="{{ postgres.db_admin_password }}" \
+                  db="{{ postgres.db_name }}"
+  tags: createuser
+
+- name: Ensure user has access to the superset database
+  postgresql_user: name="{{ postgres.db_username }}" \
+                  password="{{ postgres.db_password }}" \
+                  no_password_changes=true \
+                  priv=ALL \
+                  state=present \
+                  login_host="{{ postgres.db_url }}" \
+                  port="{{ postgres.db_port }}" \
+                  login_user="{{ postgres.db_admin_user }}" \
+                  login_password="{{ postgres.db_admin_password }}" \
+                  db="superset"
+  tags: createuser
+
+- name: create {{ env }}_consumer-channel-mapping
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_consumer_channel_mapping(consumer_id text, channel text, status smallint, created_by text, created_on timestamptz, updated_on timestamptz, PRIMARY KEY(consumer_id, channel))"
+  run_once: true
+  tags: create
+
+- name: update {{ env }}_consumer_channel_mapping
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "insert into {{ env }}_consumer_channel_mapping (consumer_id, channel, status, created_by, created_on, updated_on) values ('{{ consumer_id }}', '{{ channel }}', {{ status }}, '{{created_by }}', current_timestamp, current_timestamp) on conflict on constraint {{ env }}_consumer_channel_mapping_pkey DO UPDATE set status=EXCLUDED.status, updated_on= current_timestamp"
+  run_once: true
+  tags: insert
+
+- name: create {{ env }}_device_profile
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_device_profile(device_id text, api_last_updated_on timestamptz, avg_ts float, city text, country text, country_code text, device_spec json, district_custom text, fcm_token text, first_access timestamptz, last_access timestamptz, producer_id text, state text, state_code text, state_code_custom text, state_custom text, total_launches bigint, total_ts float, uaspec json, updated_date timestamptz, user_declared_district text, user_declared_state text, user_declared_on timestamptz, PRIMARY KEY(device_id))"
+  run_once: true
+  tags: create
+
+- name: create {{ env }}_report_config
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_report_config(report_id text, updated_on timestamptz,report_description text,requested_by text,report_schedule text,config json,created_on timestamptz,submitted_on timestamptz,status text,status_msg text,PRIMARY KEY(report_id))"
+  run_once: true
+  tags: create
+
+- name: update {{ env }}_report_config
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_report_config ADD COLUMN IF NOT EXISTS batch_number INTEGER"
+  run_once: true
+  tags: update  
+
+- name: create report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report(reportid varchar(40) NOT NULL PRIMARY KEY, title text NOT NULL, description text NOT NULL, authorizedroles jsonb NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', type varchar(8) NOT NULL CHECK (type in ('public', 'private')) DEFAULT 'private', reportaccessurl text NOT NULL UNIQUE, createdon timestamptz NOT NULL DEFAULT now(), updatedon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, reportconfig jsonb NOT NULL, templateurl text, slug varchar(10) NOT NULL, reportgenerateddate timestamptz NOT NULL DEFAULT now(), reportduration jsonb NOT NULL DEFAULT jsonb_build_object('startDate', now()::timestamptz, 'endDate', now()::timestamptz), tags jsonb NOT NULL, updatefrequency text NOT NULL)"
+  run_once: true
+  tags: create
+
+- name: create report_summary
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_summary(id varchar(40) NOT NULL PRIMARY KEY, reportid varchar(40) NOT NULL, chartid text, createdon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, summary text NOT NULL)"
+  run_once: true
+  tags: create
+
+- name: create {{ env }}_consumer-channel-mapping
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "select * from {{ env }}_consumer_channel_mapping"
+  run_once: true
+  tags: select
+
+- name: create report_status
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_status(reportid varchar(40) NOT NULL REFERENCES report(reportid) ON DELETE CASCADE, hashed_val text NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', PRIMARY KEY (reportid, hashed_val))"
+  run_once: true
+  tags: create
+
+- name: create {{ env }}_job_request
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_job_request(tag VARCHAR(100), request_id VARCHAR(50), job_id VARCHAR(50), status VARCHAR(50), request_data json, requested_by VARCHAR(50), requested_channel VARCHAR(50), dt_job_submitted TIMESTAMP, download_urls text[], dt_file_created TIMESTAMP, dt_job_completed TIMESTAMP, execution_time INTEGER, err_message VARCHAR(300), iteration INTEGER, encryption_key VARCHAR(50), PRIMARY KEY (tag, request_id))"
+  run_once: true
+  tags: create
+
+- name: update {{ env }}_job_request
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_job_request ADD COLUMN IF NOT EXISTS batch_number INTEGER"
+  run_once: true
+  tags: update
+
+- name: update {{ env }}_job_request
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_job_request ADD COLUMN IF NOT EXISTS processed_batches text"
+  run_once: true
+  tags: update      
+
+- name: create {{ env }}_experiment_definition
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_experiment_definition (exp_id VARCHAR(50), created_by VARCHAR(50), created_on TIMESTAMP, criteria VARCHAR(100), exp_data VARCHAR(300), exp_description VARCHAR(200), exp_name VARCHAR(50), stats VARCHAR(300), status VARCHAR(50), status_message VARCHAR(50), updated_by VARCHAR(50), updated_on TIMESTAMP, PRIMARY KEY(exp_id))"
+  run_once: true
+  tags: create  
+
+- name: create {{ env }}_dataset_metadata
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_dataset_metadata(dataset_id VARCHAR(50), dataset_sub_id VARCHAR(150), dataset_config json, visibility VARCHAR(50), dataset_type VARCHAR(50), version VARCHAR(10), authorized_roles text[], available_from TIMESTAMP, sample_request text, sample_response text, validation_json json, druid_query json, limits json, supported_formats text[], exhaust_type VARCHAR(50), PRIMARY KEY (dataset_id, dataset_sub_id))"
+  run_once: true
+  tags: update
+
+- name: update {{ env }}_dataset_metadata
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE {{ env }}_dataset_metadata ALTER COLUMN dataset_sub_id TYPE VARCHAR(150)"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS parameters jsonb"
+  run_once: true
+  tags: update
+
+- name: update report_summary
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report_summary ADD COLUMN IF NOT EXISTS param_hash text"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS report_type varchar(8) NOT NULL DEFAULT 'report'"
+  run_once: true
+  tags: update 
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS visibilityflags jsonb"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD COLUMN IF NOT EXISTS accesspath jsonb"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report DROP CONSTRAINT report_type_check"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ADD CONSTRAINT report_type_check CHECK (type in ('public', 'private', 'protected'))"
+  run_once: true
+  tags: update
+
+- name: update report
+  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "ALTER TABLE report ALTER COLUMN type TYPE varchar(10)"
+  run_once: true
+  tags: update
\ No newline at end of file
diff --git a/ansible/roles/postgres-managed-service-restore/defaults/main.yml b/ansible/roles/postgres-managed-service-restore/defaults/main.yml
new file mode 100644
index 0000000000..dc1cefe5bc
--- /dev/null
+++ b/ansible/roles/postgres-managed-service-restore/defaults/main.yml
@@ -0,0 +1,15 @@
+postgresql_restore_dir: /tmp/postgres-restore
+
+db:
+  name: ['analytics']
+  role: ['analytics']
+  user: ['azure_superuser, analytics']
+
+#these variables are passed as extra vars
+postgres_backup_filename:
+postgres_user: 
+postgres_password:
+postgres_hostname: 
+
+cloud_storage_dppostgresqlbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dppostgresqlbackup_foldername: dp-postgresql-backup
diff --git a/ansible/roles/postgres-managed-service-restore/tasks/main.yml b/ansible/roles/postgres-managed-service-restore/tasks/main.yml
new file mode 100644
index 0000000000..c46e43e647
--- /dev/null
+++ b/ansible/roles/postgres-managed-service-restore/tasks/main.yml
@@ -0,0 +1,99 @@
+- name: install psycopg2
+  package:
+    name: python-psycopg2
+    state: present
+   
+- name: ensure restore dir exists
+  file: path="{{ postgresql_restore_dir }}" state=directory mode=0777
+
+- set_fact:
+    postgres_backup_filepath: "{{ postgresql_restore_dir }}/{{ postgres_backup_filename }}"
+
+- name: download a file from azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dppostgresqlbackup_foldername }}"
+    blob_file_name: "{{ postgres_backup_filename }}"
+    local_file_or_folder_path: "{{ postgres_restore_dir }}/{{ postgres_backup_filepath }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+  
+- name: download a file from aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ postgres_restore_dir }}/{{ postgres_backup_filepath }}"
+    s3_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgres_backup_filename }}"
+  when: cloud_service_provider == "aws"
+   
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgres_backup_filename }}"
+    local_file_or_folder_path: "{{ postgres_restore_dir }}/{{ postgres_backup_filepath }}"
+  when: cloud_service_provider == "gcloud"
+
+- name: unarchive artifact
+  unarchive: src={{ postgresql_restore_dir }}/{{ postgres_backup_filename }} dest={{ postgresql_restore_dir }}/ copy=no
+
+- name: Create db's
+  postgresql_db:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+    state: present
+  with_items: "{{ db.name }}"
+  async: 1000
+  poll: 10
+
+- name: Create role and grant access to db's
+  postgresql_user:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    db: "{{ item[0] }}"
+    name: "{{ item[1] }}"
+    priv: ALL
+    state: present
+    role_attr_flags: CREATEROLE
+  with_nested:
+    - "{{ db.name }}"
+    - "{{ db.role }}"
+  async: 1000
+  poll: 10
+
+- name: create user
+  postgresql_user:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+  with_items: "{{ db.user }}"
+  async: 1000
+  poll: 10
+
+- name: Restore db's
+  postgresql_db:
+    login_user: "{{ postgres_user }}"
+    login_password: "{{ postgres_password }}"
+    login_host: "{{ postgres_hostname }}"
+    name: "{{ item }}"
+    state: restore
+    target: "{{ item }}.sql"
+  args:
+    chdir: "{{ postgres_restore_dir }}"
+  with_items: "{{ db.name }}"
diff --git a/ansible/roles/postgres-managed-service/defaults/main.yml b/ansible/roles/postgres-managed-service/defaults/main.yml
new file mode 100644
index 0000000000..b904c84887
--- /dev/null
+++ b/ansible/roles/postgres-managed-service/defaults/main.yml
@@ -0,0 +1,7 @@
+postgresql_user: postgres
+postgresql_backup_dir: /tmp/dp-postgres
+db_name:
+  db: ['analytics']
+
+cloud_storage_dppostgresqlbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dppostgresqlbackup_foldername: dp-postgresql-backup
diff --git a/ansible/roles/postgres-managed-service/tasks/main.yml b/ansible/roles/postgres-managed-service/tasks/main.yml
new file mode 100644
index 0000000000..c4189f2a08
--- /dev/null
+++ b/ansible/roles/postgres-managed-service/tasks/main.yml
@@ -0,0 +1,70 @@
+- name: install psycopg2
+  package:
+    name: python-psycopg2
+    state: present
+- name: ensure backup dir exists
+  file: path="{{ postgresql_backup_dir }}" state=directory mode=0777
+
+- set_fact:
+    postgresql_backup_gzip_file_name: "postgresql_backup_{{ lookup('pipe', 'date +%Z-%Y-%m-%d-%H-%M-%S') }}"
+
+- set_fact:
+    postgresql_backup_gzip_file_path: "{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}.zip"
+
+- name: Dump an existing database to a file
+  postgresql_db:
+    login_user: "{{ postgres.db_username }}"
+    login_password: "{{ postgres.db_password }}"
+    login_host: "{{ postgres.db_url }}"
+    name: "{{ item }}"
+    state: dump
+    target: "{{ postgresql_backup_dir }}/{{ item }}.sql"
+  with_items: "{{ db_name.db }}"
+  async: 3600
+  poll: 10
+
+- name: Create archive of backup directory
+  archive: path="{{ postgresql_backup_dir }}/*" dest="{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}.zip" format=zip
+  async: 500
+  poll: 10
+
+- name: upload file to azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-upload.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dppostgresqlbackup_foldername }}"
+    container_public_access: "off"
+    blob_file_name: "{{ postgresql_backup_gzip_file_name }}.zip"
+    local_file_or_folder_path: "{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}.zip"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"  
+
+- name: upload file to aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
+    s3_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}.zip"
+  when: cloud_service_provider == "aws"
+   
+- name: upload file to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}.zip"
+    local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
+  when: cloud_service_provider == "gcloud"
+
+- name: clean up backup dir after upload
+  file: path="{{ postgresql_backup_dir }}" state=absent
diff --git a/ansible/roles/postgres-provision/LICENSE b/ansible/roles/postgres-provision/LICENSE
deleted file mode 100644
index 4275cf3c10..0000000000
--- a/ansible/roles/postgres-provision/LICENSE
+++ /dev/null
@@ -1,20 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2017 Jeff Geerling
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
-the Software, and to permit persons to whom the Software is furnished to do so,
-subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
-FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
-COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
-IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/ansible/roles/postgres-provision/README.md b/ansible/roles/postgres-provision/README.md
index 6d81e4e9f9..a30ec22270 100644
--- a/ansible/roles/postgres-provision/README.md
+++ b/ansible/roles/postgres-provision/README.md
@@ -1,6 +1,6 @@
 # Ansible Role: PostgreSQL
 
-[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-postgresql.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-postgresql)
+[![CI](https://github.com/geerlingguy/ansible-role-postgresql/workflows/CI/badge.svg?event=push)](https://github.com/geerlingguy/ansible-role-postgresql/actions?query=workflow%3ACI)
 
 Installs and configures PostgreSQL server on RHEL/CentOS or Debian/Ubuntu servers.
 
@@ -106,6 +106,10 @@ A list of databases to ensure exist on the server. Only the `name` is required;
 
 A list of users to ensure exist on the server. Only the `name` is required; all other properties are optional.
 
+    postgres_users_no_log: true
+
+Whether to output user data (which may contain sensitive information, like passwords) when managing users.
+
     postgresql_version: [OS-specific]
     postgresql_data_dir: [OS-specific]
     postgresql_bin_path: [OS-specific]
diff --git a/ansible/roles/postgres-provision/defaults/main.yml b/ansible/roles/postgres-provision/defaults/main.yml
index 73e40f0125..8ea9c992e4 100644
--- a/ansible/roles/postgres-provision/defaults/main.yml
+++ b/ansible/roles/postgres-provision/defaults/main.yml
@@ -10,6 +10,9 @@ postgresql_python_library: python-psycopg2
 postgresql_user: postgres
 postgresql_group: postgres
 
+# `md5` or `scram-sha-256` (https://www.postgresql.org/docs/10/auth-methods.html)
+postgresql_auth_method: "{{ ansible_fips  | ternary('scram-sha-256', 'md5') }}"
+
 postgresql_unix_socket_directories:
   - /var/run/postgresql
 
@@ -24,10 +27,11 @@ postgresql_global_config_options:
 # Host based authentication (hba) entries to be added to the pg_hba.conf. This
 # variable's defaults reflect the defaults that come with a fresh installation.
 postgresql_hba_entries:
-  - { type: local, database: all, user: postgres, auth_method: peer }
-  - { type: local, database: all, user: all, auth_method: peer }
-  - { type: host, database: all, user: all, address: '{{ postgres_address_space }}', auth_method: md5 }
-  - { type: host, database: all, user: all, address: '::1/128', auth_method: md5 }
+  - {type: local, database: all, user: postgres, auth_method: peer}
+  - {type: local, database: all, user: all, auth_method: peer}
+  - {type: host, database: all, user: all, address: '127.0.0.1/32', auth_method: "{{ postgresql_auth_method }}"}
+  - {type: host, database: all, user: all, address: '::1/128', auth_method: "{{ postgresql_auth_method }}"}
+  - {type: host, database: all, user: all, address: '{{ postgres_address_space }}', auth_method: md5}
 
 # Debian only. Used to generate the locales used by PostgreSQL databases.
 postgresql_locales:
@@ -62,3 +66,6 @@ postgresql_users: []
 #   login_unix_socket: # defaults to 1st of postgresql_unix_socket_directories
 #   port: # defaults to not set
 #   state: # defaults to 'present'
+
+# Whether to output user data when managing users.
+postgres_users_no_log: true
diff --git a/ansible/roles/postgres-provision/meta/.galaxy_install_info b/ansible/roles/postgres-provision/meta/.galaxy_install_info
deleted file mode 100644
index fa7f3604a7..0000000000
--- a/ansible/roles/postgres-provision/meta/.galaxy_install_info
+++ /dev/null
@@ -1 +0,0 @@
-{install_date: 'Thu Oct 11 13:38:42 2018', version: 1.4.4}
diff --git a/ansible/roles/postgres-provision/meta/main.yml b/ansible/roles/postgres-provision/meta/main.yml
deleted file mode 100644
index ccdd8577d1..0000000000
--- a/ansible/roles/postgres-provision/meta/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
----
-dependencies: []
-
-galaxy_info:
-  author: geerlingguy
-  description: PostgreSQL server for Linux.
-  company: "Midwestern Mac, LLC"
-  license: "license (BSD, MIT)"
-  min_ansible_version: 2.4
-  platforms:
-    - name: EL
-      versions:
-        - 6
-        - 7
-    - name: Ubuntu
-      versions:
-        - all
-    - name: Debian
-      versions:
-        - all
-  galaxy_tags:
-    - database
-    - postgresql
-    - postgres
-    - rdbms
diff --git a/ansible/roles/postgres-provision/molecule/default/molecule.yml b/ansible/roles/postgres-provision/molecule/default/molecule.yml
deleted file mode 100644
index 033970241f..0000000000
--- a/ansible/roles/postgres-provision/molecule/default/molecule.yml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-dependency:
-  name: galaxy
-driver:
-  name: docker
-lint:
-  name: yamllint
-  options:
-    config-file: molecule/default/yaml-lint.yml
-platforms:
-  - name: instance
-    image: geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible
-    command: ${MOLECULE_DOCKER_COMMAND:-"sleep infinity"}
-    privileged: true
-    pre_build_image: true
-provisioner:
-  name: ansible
-  lint:
-    name: ansible-lint
-  playbooks:
-    converge: ${MOLECULE_PLAYBOOK:-playbook.yml}
-scenario:
-  name: default
-verifier:
-  name: testinfra
-  lint:
-    name: flake8
diff --git a/ansible/roles/postgres-provision/molecule/default/playbook.yml b/ansible/roles/postgres-provision/molecule/default/playbook.yml
deleted file mode 100644
index 5f152a6cab..0000000000
--- a/ansible/roles/postgres-provision/molecule/default/playbook.yml
+++ /dev/null
@@ -1,35 +0,0 @@
----
-- name: Converge
-  hosts: all
-  become: true
-
-  vars:
-    postgresql_databases:
-      - name: example
-    postgresql_users:
-      - name: jdoe
-
-  pre_tasks:
-    - name: Update apt cache.
-      apt: update_cache=true cache_valid_time=600
-      when: ansible_os_family == 'Debian'
-
-    - name: Set custom variables for old CentOS 6 PostgreSQL install.
-      set_fact:
-        postgresql_hba_entries: []
-        postgresql_global_config_options:
-          - option: unix_socket_directory
-            value: '{{ postgresql_unix_socket_directories[0] }}'
-      when:
-        - ansible_os_family == 'RedHat'
-        - ansible_distribution_version.split('.')[0] == '6'
-
-  roles:
-    - role: geerlingguy.postgresql
-
-  post_tasks:
-    - name: Verify postgres is running.
-      command: "{{ postgresql_bin_path }}/pg_ctl -D {{ postgresql_data_dir }} status"
-      changed_when: false
-      become: true
-      become_user: postgres
diff --git a/ansible/roles/postgres-provision/molecule/default/tests/test_default.py b/ansible/roles/postgres-provision/molecule/default/tests/test_default.py
deleted file mode 100644
index eedd64a1d3..0000000000
--- a/ansible/roles/postgres-provision/molecule/default/tests/test_default.py
+++ /dev/null
@@ -1,14 +0,0 @@
-import os
-
-import testinfra.utils.ansible_runner
-
-testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-    os.environ['MOLECULE_INVENTORY_FILE']).get_hosts('all')
-
-
-def test_hosts_file(host):
-    f = host.file('/etc/hosts')
-
-    assert f.exists
-    assert f.user == 'root'
-    assert f.group == 'root'
diff --git a/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml b/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml
deleted file mode 100644
index a3dbc38eef..0000000000
--- a/ansible/roles/postgres-provision/molecule/default/yaml-lint.yml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-extends: default
-rules:
-  line-length:
-    max: 120
-    level: warning
diff --git a/ansible/roles/postgres-provision/tasks/configure.yml b/ansible/roles/postgres-provision/tasks/configure.yml
index e542b12952..df4346450f 100644
--- a/ansible/roles/postgres-provision/tasks/configure.yml
+++ b/ansible/roles/postgres-provision/tasks/configure.yml
@@ -5,24 +5,19 @@
     regexp: "^#?{{ item.option }}.+$"
     line: "{{ item.option }} = '{{ item.value }}'"
     state: "{{ item.state | default('present') }}"
+    mode: 0644
   with_items: "{{ postgresql_global_config_options }}"
   notify: restart postgresql
 
-- name: Configure global settings.
-  lineinfile:
-    dest: "{{ postgresql_config_path }}/postgresql.conf"
-    line: "listen_addresses = '*'"
-  notify: restart postgresql
-
 - name: Configure host based authentication (if entries are configured).
   template:
-    src: "templates/pg_hba.conf.j2"
+    src: "pg_hba.conf.j2"
     dest: "{{ postgresql_config_path }}/pg_hba.conf"
     owner: "{{ postgresql_user }}"
     group: "{{ postgresql_group }}"
     mode: 0600
   notify: restart postgresql
-  when: postgresql_hba_entries
+  when: postgresql_hba_entries | length > 0
 
 - name: Ensure PostgreSQL unix socket dirs exist.
   file:
@@ -30,5 +25,5 @@
     state: directory
     owner: "{{ postgresql_user }}"
     group: "{{ postgresql_group }}"
-    mode: 02775
+    mode: "{{ postgresql_unix_socket_directories_mode }}"
   with_items: "{{ postgresql_unix_socket_directories }}"
diff --git a/ansible/roles/postgres-provision/tasks/databases.yml b/ansible/roles/postgres-provision/tasks/databases.yml
index e01d804d47..d13313e58c 100644
--- a/ansible/roles/postgres-provision/tasks/databases.yml
+++ b/ansible/roles/postgres-provision/tasks/databases.yml
@@ -2,8 +2,6 @@
 - name: Ensure PostgreSQL databases are present.
   postgresql_db:
     name: "{{ item.name }}"
-    lc_collate: "{{ item.lc_collate | default('en_US.UTF-8') }}"
-    lc_ctype: "{{ item.lc_ctype | default('en_US.UTF-8') }}"
     encoding: "{{ item.encoding | default('UTF-8') }}"
     template: "{{ item.template | default('template0') }}"
     login_host: "{{ item.login_host | default('localhost') }}"
diff --git a/ansible/roles/postgres-provision/tasks/main.yml b/ansible/roles/postgres-provision/tasks/main.yml
index d2536265c6..308f38e38a 100644
--- a/ansible/roles/postgres-provision/tasks/main.yml
+++ b/ansible/roles/postgres-provision/tasks/main.yml
@@ -1,16 +1,16 @@
 ---
 # Variable configuration.
-- include: variables.yml
+- include_tasks: variables.yml
 
 # Setup/install tasks.
-- include: setup-RedHat.yml
+- include_tasks: setup-RedHat.yml
   when: ansible_os_family == 'RedHat'
 
-- include: setup-Debian.yml
+- include_tasks: setup-Debian.yml
   when: ansible_os_family == 'Debian'
 
-- include: initialize.yml
-- include: configure.yml
+- include_tasks: initialize.yml
+- include_tasks: configure.yml
 
 - name: Ensure PostgreSQL is started and enabled on boot.
   service:
@@ -19,11 +19,6 @@
     enabled: "{{ postgresql_service_enabled }}"
 
 # Configure PostgreSQL.
-- include: users.yml
-- include: databases.yml
-- name: Ensure PostgreSQL is started and enabled on boot.
-  service:
-    name: "{{ postgresql_daemon }}"
-    state: started
-    enabled: yes
-
+- import_tasks: users.yml
+- import_tasks: databases.yml
+- import_tasks: users_props.yml
diff --git a/ansible/roles/postgres-provision/tasks/setup-Debian.yml b/ansible/roles/postgres-provision/tasks/setup-Debian.yml
index fc47d0f02a..1b540196f0 100644
--- a/ansible/roles/postgres-provision/tasks/setup-Debian.yml
+++ b/ansible/roles/postgres-provision/tasks/setup-Debian.yml
@@ -6,9 +6,8 @@
 
 - name: Ensure PostgreSQL packages are installed.
   apt:
-    name: "{{ item }}"
+    name: "{{ postgresql_packages }}"
     state: present
-  with_items: "{{ postgresql_packages }}"
 
 - name: Ensure all configured locales are present.
   locale_gen: "name={{ item }} state=present"
diff --git a/ansible/roles/postgres-provision/tasks/setup-RedHat.yml b/ansible/roles/postgres-provision/tasks/setup-RedHat.yml
index 9cd84ed01a..d536bcb012 100644
--- a/ansible/roles/postgres-provision/tasks/setup-RedHat.yml
+++ b/ansible/roles/postgres-provision/tasks/setup-RedHat.yml
@@ -1,12 +1,16 @@
 ---
 - name: Ensure PostgreSQL packages are installed.
-  package:
+  yum:
     name: "{{ postgresql_packages }}"
     state: present
     enablerepo: "{{ postgresql_enablerepo | default(omit, true) }}"
+    # Don't let postgresql-contrib cause the /usr/bin/python symlink
+    # to be installed, which breaks later Ansible runs on Fedora 30,
+    # and affects system behavior in multiple ways.
+    exclude: python-unversioned-command
 
 - name: Ensure PostgreSQL Python libraries are installed.
-  package:
+  yum:
     name: "{{ postgresql_python_library }}"
     state: present
     enablerepo: "{{ postgresql_enablerepo | default(omit, true) }}"
diff --git a/ansible/roles/postgres-provision/tasks/users.yml b/ansible/roles/postgres-provision/tasks/users.yml
index 34746ebe9d..4d346a9c81 100644
--- a/ansible/roles/postgres-provision/tasks/users.yml
+++ b/ansible/roles/postgres-provision/tasks/users.yml
@@ -3,20 +3,12 @@
   postgresql_user:
     name: "{{ item.name }}"
     password: "{{ item.password | default(omit) }}"
-    encrypted: "{{ item.encrypted | default(omit) }}"
-    priv: "{{ item.priv | default(omit) }}"
-    role_attr_flags: "{{ item.role_attr_flags | default(omit) }}"
-    db: "{{ item.db | default(omit) }}"
-    login_host: "{{ item.login_host | default('localhost') }}"
-    login_password: "{{ item.login_password | default(omit) }}"
-    login_user: "{{ item.login_user | default(postgresql_user) }}"
-    login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}"
-    port: "{{ item.port | default(omit) }}"
-    state: "{{ item.state | default('present') }}"
   with_items: "{{ postgresql_users }}"
-  no_log: true
+  no_log: "{{ postgres_users_no_log }}"
   become: true
   become_user: "{{ postgresql_user }}"
   # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509
   vars:
     ansible_ssh_pipelining: true
+  environment:
+    PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}"
diff --git a/ansible/roles/postgres-provision/tasks/users_props.yml b/ansible/roles/postgres-provision/tasks/users_props.yml
new file mode 100644
index 0000000000..992ccf5c9e
--- /dev/null
+++ b/ansible/roles/postgres-provision/tasks/users_props.yml
@@ -0,0 +1,24 @@
+---
+- name: Ensure PostgreSQL users are configured correctly.
+  postgresql_user:
+    name: "{{ item.name }}"
+    password: "{{ item.password | default(omit) }}"
+    encrypted: "{{ item.encrypted | default(omit) }}"
+    priv: "{{ item.priv | default(omit) }}"
+    role_attr_flags: "{{ item.role_attr_flags | default(omit) }}"
+    db: "{{ item.db | default(omit) }}"
+    login_host: "{{ item.login_host | default('localhost') }}"
+    login_password: "{{ item.login_password | default(omit) }}"
+    login_user: "{{ item.login_user | default(postgresql_user) }}"
+    login_unix_socket: "{{ item.login_unix_socket | default(postgresql_unix_socket_directories[0]) }}"
+    port: "{{ item.port | default(omit) }}"
+    state: "{{ item.state | default('present') }}"
+  with_items: "{{ postgresql_users }}"
+  no_log: "{{ postgres_users_no_log }}"
+  become: true
+  become_user: "{{ postgresql_user }}"
+  # See: https://github.com/ansible/ansible/issues/16048#issuecomment-229012509
+  vars:
+    ansible_ssh_pipelining: true
+  environment:
+    PGOPTIONS: "{{ (postgresql_auth_method == 'scram-sha-256') | ternary('-c password_encryption=scram-sha-256', '') }}"
diff --git a/ansible/roles/postgres-provision/tasks/variables.yml b/ansible/roles/postgres-provision/tasks/variables.yml
index 530990d7ea..575897207f 100644
--- a/ansible/roles/postgres-provision/tasks/variables.yml
+++ b/ansible/roles/postgres-provision/tasks/variables.yml
@@ -6,23 +6,29 @@
 
 - name: Include OS-specific variables (RedHat).
   include_vars: "{{ ansible_os_family }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
-  when: ansible_os_family == 'RedHat'
+  when:
+  - ansible_os_family == 'RedHat'
+  - ansible_distribution != 'Fedora'
+
+- name: Include OS-specific variables (Fedora).
+  include_vars: "{{ ansible_distribution }}-{{ ansible_distribution_version.split('.')[0] }}.yml"
+  when: ansible_distribution == 'Fedora'
 
 - name: Define postgresql_packages.
   set_fact:
     postgresql_packages: "{{ __postgresql_packages | list }}"
   when: postgresql_packages is not defined
 
-- name: Define postgresql_daemon.
-  set_fact:
-    postgresql_daemon: "{{ __postgresql_daemon }}"
-  when: postgresql_daemon is not defined
-
 - name: Define postgresql_version.
   set_fact:
     postgresql_version: "{{ __postgresql_version }}"
   when: postgresql_version is not defined
 
+- name: Define postgresql_daemon.
+  set_fact:
+    postgresql_daemon: "{{ __postgresql_daemon }}"
+  when: postgresql_daemon is not defined
+
 - name: Define postgresql_data_dir.
   set_fact:
     postgresql_data_dir: "{{ __postgresql_data_dir }}"
@@ -37,3 +43,9 @@
   set_fact:
     postgresql_config_path: "{{ __postgresql_config_path }}"
   when: postgresql_config_path is not defined
+
+- name: Define postgresql_unix_socket_directories_mode.
+  set_fact:
+    postgresql_unix_socket_directories_mode: >-
+      {{ __postgresql_unix_socket_directories_mode | default('02775') }}
+  when: postgresql_unix_socket_directories_mode is not defined
diff --git a/ansible/roles/postgres-provision/vars/Debian-10.yml b/ansible/roles/postgres-provision/vars/Debian-10.yml
new file mode 100644
index 0000000000..d8b5103b6f
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Debian-10.yml
@@ -0,0 +1,12 @@
+---
+__postgresql_version: "11"
+__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main"
+__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin"
+__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main"
+__postgresql_daemon: "postgresql@{{ postgresql_version }}-main"
+__postgresql_packages:
+  - postgresql
+  - postgresql-contrib
+  - libpq-dev
+# Debian 10 uses Python 3 by default.
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Debian-8.yml b/ansible/roles/postgres-provision/vars/Debian-8.yml
index f6ad232b51..ec86f938fb 100644
--- a/ansible/roles/postgres-provision/vars/Debian-8.yml
+++ b/ansible/roles/postgres-provision/vars/Debian-8.yml
@@ -3,7 +3,7 @@ __postgresql_version: "9.4"
 __postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main"
 __postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin"
 __postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main"
-__postgresql_daemon: postgresql
+__postgresql_daemon: "postgresql@{{ postgresql_version }}-main"
 __postgresql_packages:
   - postgresql
   - postgresql-contrib
diff --git a/ansible/roles/postgres-provision/vars/Debian-9.yml b/ansible/roles/postgres-provision/vars/Debian-9.yml
index 7b57d53241..2afb9f42f0 100644
--- a/ansible/roles/postgres-provision/vars/Debian-9.yml
+++ b/ansible/roles/postgres-provision/vars/Debian-9.yml
@@ -3,7 +3,7 @@ __postgresql_version: "9.6"
 __postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main"
 __postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin"
 __postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main"
-__postgresql_daemon: postgresql
+__postgresql_daemon: "postgresql@{{ postgresql_version }}-main"
 __postgresql_packages:
   - postgresql
   - postgresql-contrib
diff --git a/ansible/roles/postgres-provision/vars/Fedora-29.yml b/ansible/roles/postgres-provision/vars/Fedora-29.yml
new file mode 100644
index 0000000000..4e099324bf
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Fedora-29.yml
@@ -0,0 +1,12 @@
+---
+__postgresql_version: "10.5"
+__postgresql_data_dir: "/var/lib/pgsql/data"
+__postgresql_bin_path: "/usr/bin"
+__postgresql_config_path: "/var/lib/pgsql/data"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-server
+  - postgresql-contrib
+  - postgresql-libs
+postgresql_python_library: python2-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Fedora-30.yml b/ansible/roles/postgres-provision/vars/Fedora-30.yml
new file mode 100644
index 0000000000..d07f14b7bd
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Fedora-30.yml
@@ -0,0 +1,13 @@
+---
+__postgresql_version: "11.2"
+__postgresql_data_dir: "/var/lib/pgsql/data"
+__postgresql_bin_path: "/usr/bin"
+__postgresql_config_path: "/var/lib/pgsql/data"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-server
+  - postgresql-contrib
+  - postgresql-libs
+# Fedora 30 containers only have python3 by default
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Fedora-31.yml b/ansible/roles/postgres-provision/vars/Fedora-31.yml
new file mode 100644
index 0000000000..27a023eccd
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Fedora-31.yml
@@ -0,0 +1,14 @@
+---
+__postgresql_version: "11.5"
+__postgresql_data_dir: "/var/lib/pgsql/data"
+__postgresql_bin_path: "/usr/bin"
+__postgresql_config_path: "/var/lib/pgsql/data"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-server
+  - postgresql-contrib
+  - postgresql-libs
+__postgresql_unix_socket_directories_mode: '0755'
+# Fedora 31 containers only have python3 by default
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Fedora-32.yml b/ansible/roles/postgres-provision/vars/Fedora-32.yml
new file mode 100644
index 0000000000..6ce9d9bfe4
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Fedora-32.yml
@@ -0,0 +1,14 @@
+---
+__postgresql_version: "12.2"
+__postgresql_data_dir: "/var/lib/pgsql/data"
+__postgresql_bin_path: "/usr/bin"
+__postgresql_config_path: "/var/lib/pgsql/data"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-server
+  - postgresql-contrib
+  - postgresql-libs
+__postgresql_unix_socket_directories_mode: '0755'
+# Fedora 32 containers only have python3 by default
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/RedHat-6.yml b/ansible/roles/postgres-provision/vars/RedHat-6.yml
deleted file mode 100644
index 8923c50dd9..0000000000
--- a/ansible/roles/postgres-provision/vars/RedHat-6.yml
+++ /dev/null
@@ -1,11 +0,0 @@
----
-__postgresql_version: "8.4"
-__postgresql_data_dir: "/var/lib/pgsql/data"
-__postgresql_bin_path: "/usr/bin"
-__postgresql_config_path: "/var/lib/pgsql/data"
-__postgresql_daemon: postgresql
-__postgresql_packages:
-  - postgresql
-  - postgresql-server
-  - postgresql-contrib
-  - postgresql-libs
diff --git a/ansible/roles/postgres-provision/vars/RedHat-8.yml b/ansible/roles/postgres-provision/vars/RedHat-8.yml
new file mode 100644
index 0000000000..e519ea9a08
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/RedHat-8.yml
@@ -0,0 +1,12 @@
+---
+__postgresql_version: "10"
+__postgresql_data_dir: "/var/lib/pgsql/data"
+__postgresql_bin_path: "/usr/bin"
+__postgresql_config_path: "/var/lib/pgsql/data"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-server
+  - postgresql-contrib
+__postgresql_unix_socket_directories_mode: '0755'
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-14.yml b/ansible/roles/postgres-provision/vars/Ubuntu-14.yml
deleted file mode 100644
index bd6c174757..0000000000
--- a/ansible/roles/postgres-provision/vars/Ubuntu-14.yml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-__postgresql_version: "9.3"
-__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main"
-__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin"
-__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main"
-__postgresql_daemon: postgresql
-__postgresql_packages:
-  - postgresql
-  - postgresql-contrib
-  - libpq-dev
diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-18.yml b/ansible/roles/postgres-provision/vars/Ubuntu-18.yml
index 201acc365e..8136224f2b 100644
--- a/ansible/roles/postgres-provision/vars/Ubuntu-18.yml
+++ b/ansible/roles/postgres-provision/vars/Ubuntu-18.yml
@@ -8,3 +8,4 @@ __postgresql_packages:
   - postgresql
   - postgresql-contrib
   - libpq-dev
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres-provision/vars/Ubuntu-20.yml b/ansible/roles/postgres-provision/vars/Ubuntu-20.yml
new file mode 100644
index 0000000000..9a9a065a28
--- /dev/null
+++ b/ansible/roles/postgres-provision/vars/Ubuntu-20.yml
@@ -0,0 +1,11 @@
+---
+__postgresql_version: "12"
+__postgresql_data_dir: "/var/lib/postgresql/{{ __postgresql_version }}/main"
+__postgresql_bin_path: "/usr/lib/postgresql/{{ __postgresql_version }}/bin"
+__postgresql_config_path: "/etc/postgresql/{{ __postgresql_version }}/main"
+__postgresql_daemon: postgresql
+__postgresql_packages:
+  - postgresql
+  - postgresql-contrib
+  - libpq-dev
+postgresql_python_library: python3-psycopg2
diff --git a/ansible/roles/postgres/tasks/main.yml b/ansible/roles/postgres/tasks/main.yml
deleted file mode 100644
index 9da7f6a158..0000000000
--- a/ansible/roles/postgres/tasks/main.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-- name: Install PostgreSQL client
-  apt: name={{ item }} update_cache=yes cache_valid_time=36000 state=installed
-  with_items:
-    - python-psycopg2
-    - postgresql-client
-  ignore_errors: yes
-
-- name: Ensure database is created
-  postgresql_db: name="{{ postgres.db_name }}" \
-                login_host="{{ postgres.db_url }}" \
-                port="{{ postgres.db_port }}" \
-                login_user="{{ postgres.db_admin_user }}" \
-                login_password="{{ postgres.db_admin_password }}" \
-                encoding='UTF-8' \
-                state=present
-  tags: createdb
-
-- name: Ensure user has access to the database
-  postgresql_user: name="{{ postgres.db_username }}" \
-                  password="{{ postgres.db_password }}" \
-                  no_password_changes=true \
-                  priv=ALL \
-                  state=present \
-                  login_host="{{ postgres.db_url }}" \
-                  port="{{ postgres.db_port }}" \
-                  login_user="{{ postgres.db_admin_user }}" \
-                  login_password="{{ postgres.db_admin_password }}" \
-                  db="{{ postgres.db_name }}"
-  tags: createuser
-
-- name: create {{ env }}_consumer-channel-mapping
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_consumer_channel_mapping(consumer_id text, channel text, status smallint, created_by text, created_on timestamptz, updated_on timestamptz, PRIMARY KEY(consumer_id, channel))"
-  run_once: true
-  tags: create
-
-- name: update {{ env }}_consumer_channel_mapping
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "insert into {{ env }}_consumer_channel_mapping (consumer_id, channel, status, created_by, created_on, updated_on) values ('{{ consumer_id }}', '{{ channel }}', {{ status }}, '{{created_by }}', current_timestamp, current_timestamp) on conflict on constraint {{ env }}_consumer_channel_mapping_pkey DO UPDATE set status=EXCLUDED.status, updated_on= current_timestamp"
-  run_once: true
-  tags: insert
-
-- name: create {{ env }}_device_profile
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_device_profile(device_id text, api_last_updated_on timestamptz, avg_ts float, city text, country text, country_code text, device_spec json, district_custom text, fcm_token text, first_access timestamptz, last_access timestamptz, producer_id text, state text, state_code text, state_code_custom text, state_custom text, total_launches bigint, total_ts float, uaspec json, updated_date timestamptz, user_declared_district text, user_declared_state text, user_declared_on timestamptz, PRIMARY KEY(device_id))"
-  run_once: true
-  tags: create
-
-- name: create {{ env }}_report_config
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS {{ env }}_report_config(report_id text, updated_on timestamptz,report_description text,requested_by text,report_schedule text,config json,created_on timestamptz,submitted_on timestamptz,status text,status_msg text,PRIMARY KEY(report_id))"
-  run_once: true
-  tags: create
-
-- name: create report
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report(reportid varchar(40) NOT NULL PRIMARY KEY, title text NOT NULL, description text NOT NULL, authorizedroles jsonb NOT NULL, status varchar(8) NOT NULL CHECK (status IN ('live', 'draft', 'retired')) DEFAULT 'draft', type varchar(8) NOT NULL CHECK (type in ('public', 'private')) DEFAULT 'private', reportaccessurl text NOT NULL UNIQUE, createdon timestamptz NOT NULL DEFAULT now(), updatedon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, reportconfig jsonb NOT NULL, templateurl text, slug varchar(10) NOT NULL, reportgenerateddate timestamptz NOT NULL DEFAULT now(), reportduration jsonb NOT NULL DEFAULT jsonb_build_object('startDate', now()::timestamptz, 'endDate', now()::timestamptz), tags jsonb NOT NULL, updatefrequency text NOT NULL)"
-  run_once: true
-  tags: create
-
-- name: create report_summary
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "CREATE TABLE IF NOT EXISTS report_summary(id varchar(40) NOT NULL PRIMARY KEY, reportid varchar(40) NOT NULL, chartid text, createdon timestamptz NOT NULL DEFAULT now(), createdby varchar(50) NOT NULL, summary text NOT NULL)"
-  run_once: true
-  tags: create
-
-- name: create {{ env }}_consumer-channel-mapping
-  shell: PGPASSWORD="{{ postgres.db_password }}" psql -U "{{ postgres.db_username }}" -d "{{ postgres.db_name }}" -h "{{ postgres.db_url }}" -p "{{ postgres.db_port }}" -c "select * from {{ env }}_consumer_channel_mapping"
-  run_once: true
-  tags: select
diff --git a/ansible/roles/postgresql-backup/defaults/main.yml b/ansible/roles/postgresql-backup/defaults/main.yml
index d1b26640d1..a27674128c 100755
--- a/ansible/roles/postgresql-backup/defaults/main.yml
+++ b/ansible/roles/postgresql-backup/defaults/main.yml
@@ -1,7 +1,5 @@
 postgresql_backup_dir: /tmp/postgresql-backup
 postgresql_user: postgres
-postgresql_backup_azure_container_name: {{ azure_container }}
-postgresql_backup_azure_storage_account_name: {{sunbird_management_storage_account_name}}
-postgresql_backup_azure_storage_access_key: {{sunbird_management_storage_account_key}}
 
-# Set these vars per environment as show in example below
+cloud_storage_dppostgresqlbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dppostgresqlbackup_foldername: dp-postgresql-backup
diff --git a/ansible/roles/postgresql-backup/meta/main.yml b/ansible/roles/postgresql-backup/meta/main.yml
deleted file mode 100755
index 23b18a800a..0000000000
--- a/ansible/roles/postgresql-backup/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
\ No newline at end of file
diff --git a/ansible/roles/postgresql-backup/tasks/main.yml b/ansible/roles/postgresql-backup/tasks/main.yml
index 9e3ee22008..4be4bda8db 100755
--- a/ansible/roles/postgresql-backup/tasks/main.yml
+++ b/ansible/roles/postgresql-backup/tasks/main.yml
@@ -2,7 +2,7 @@
   file: path="{{ postgresql_backup_dir }}" state=directory owner={{ postgresql_user }} group={{ postgresql_user }}
 
 - set_fact:
-    postgresql_backup_gzip_file_name: "postgresql_backup_{{ lookup('pipe', 'date +%Y-%m-%d') }}.sql.gz"
+    postgresql_backup_gzip_file_name: "postgresql_backup_{{ lookup('pipe', 'date +%Z-%Y-%m-%d-%H-%M-%S') }}.sql.gz"
 
 - set_fact:
     postgresql_backup_gzip_file_path: "{{ postgresql_backup_dir }}/{{ postgresql_backup_gzip_file_name }}"
@@ -13,20 +13,43 @@
   async: 3600
   poll: 10
 
-- name: Ensure azure blob storage container exists
-  command: az storage container create  --name {{ postgresql_backup_azure_container_name }}
-  ignore_errors: true
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
+- name: upload file to azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-upload.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dppostgresqlbackup_foldername }}"
+    container_public_access: "off"
+    blob_file_name: "{{ postgresql_backup_gzip_file_name }}"
+    local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
 
-- name: Upload to azure blob storage
-  command: az storage blob upload --name {{ postgresql_backup_gzip_file_name }} --file {{ postgresql_backup_gzip_file_path }} --container-name {{ postgresql_backup_azure_container_name }}
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-  async: 3600
-  poll: 10
+- name: upload file to aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
+    s3_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}"
+  when: cloud_service_provider == "aws"
+     
+- name: upload file to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_backup_gzip_file_name }}"
+    local_file_or_folder_path: "{{ postgresql_backup_gzip_file_path }}"
+  when: cloud_service_provider == "gcloud"
 
 - name: clean up backup dir after upload
   file: path="{{ postgresql_backup_dir }}" state=absent
diff --git a/ansible/roles/postgresql-restore/defaults/main.yml b/ansible/roles/postgresql-restore/defaults/main.yml
index 036b005dd3..571a4f59fd 100755
--- a/ansible/roles/postgresql-restore/defaults/main.yml
+++ b/ansible/roles/postgresql-restore/defaults/main.yml
@@ -3,13 +3,6 @@ postgresql_port: 5432
 postgresql_cluster_version: 9.5
 postgresql_cluster_name: main
 postgresql_user: postgres
-postgresql_restore_azure_container_name: {{ azure_container }}
-postgresql_restore_azure_storage_account_name: {{sunbird_management_storage_account_name}}
-postgresql_restore_azure_storage_access_key: {{sunbird_management_storage_account_key}}
-#postgresql_restore_gzip_file_name: {{ backup_file_name }}
 
-# Set these vars per environment as show in example below
-# postgresql_restore_azure_storage_account_name: ntpbackupsstaging
-
-# Pass the parameter
-# postgresql_restore_gzip_file_name: <postgresql_restore_gzip_file_name>
+cloud_storage_dppostgresqlbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dppostgresqlbackup_foldername: dp-postgresql-backup
diff --git a/ansible/roles/postgresql-restore/meta/main.yml b/ansible/roles/postgresql-restore/meta/main.yml
deleted file mode 100755
index 23b18a800a..0000000000
--- a/ansible/roles/postgresql-restore/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
\ No newline at end of file
diff --git a/ansible/roles/postgresql-restore/tasks/main.yml b/ansible/roles/postgresql-restore/tasks/main.yml
index 4826a8d587..7635f7e741 100755
--- a/ansible/roles/postgresql-restore/tasks/main.yml
+++ b/ansible/roles/postgresql-restore/tasks/main.yml
@@ -2,15 +2,44 @@
   file: path="{{ postgresql_restore_dir }}" state=directory owner={{ postgresql_user }} group={{ postgresql_user }}
 
 - set_fact:
-    postgresql_restore_gzip_file_path: "{{ postgresql_restore_dir }}/postgresql_backup_{{ postgresql_restore_gzip_file_date }}.sql.gz"
-
-- name: Download restore file from azure
-  command: az storage blob download --container-name {{ postgresql_restore_azure_container_name }} --name postgresql_backup_{{ postgresql_restore_gzip_file_date }}.sql.gz --file {{ postgresql_restore_gzip_file_path }}
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-  async: 3600
-  poll: 10
+    postgresql_restore_gzip_file_path: "{{ postgresql_restore_dir }}/{{ postgresql_restore_gzip_file_name }}"
+
+- name: download a file from azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dppostgresqlbackup_foldername }}"
+    blob_file_name: "{{ postgresql_restore_gzip_file_name }}"
+    local_file_or_folder_path: "{{ postgresql_restore_gzip_file_path  }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+
+- name: download a file from aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ postgresql_restore_gzip_file_path  }}"
+    s3_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_restore_gzip_file_name }}"
+  when: cloud_service_provider == "aws"
+  
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dppostgresqlbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dppostgresqlbackup_foldername }}/{{ postgresql_restore_gzip_file_name }}"
+    local_file_or_folder_path: "{{ postgresql_restore_gzip_file_path }}"
+  when: cloud_service_provider == "gcloud"
 
 - name: ensure postgresql service is stopped
   service: name=postgresql state=stopped
@@ -18,15 +47,6 @@
 - name: wait for postgresql to be stopped
   wait_for: port={{ postgresql_port }} state=stopped
 
-- name: drop cluster
-  command: pg_dropcluster {{ postgresql_cluster_version }} {{ postgresql_cluster_name }}
-  become_user: "{{ postgresql_user }}"
-  ignore_errors: true
-
-- name: create cluster
-  command: pg_createcluster {{ postgresql_cluster_version }} {{ postgresql_cluster_name }}
-  become_user: "{{ postgresql_user }}"
-
 - name: ensure postgresql service is started
   service: name=postgresql state=started
 
diff --git a/ansible/roles/provision-azure-spark-cluster/defaults/main.yml b/ansible/roles/provision-azure-spark-cluster/defaults/main.yml
new file mode 100644
index 0000000000..5b7b50cd2f
--- /dev/null
+++ b/ansible/roles/provision-azure-spark-cluster/defaults/main.yml
@@ -0,0 +1,19 @@
+spark_folder: /usr/hdp/current/spark3-client
+guava_version: 19.0
+log4j_version: 2.16.0
+spark_redis_version: 2.5.0
+guava_default_jre_version: 28.0-jre
+guava_jre_version: 24.1.1-jre
+jedis_version: 3.2.0
+zip4j_version: 2.6.2
+guice_default_version: 4.0
+guice_version: 3.0
+guava_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_version}}/guava-{{guava_version}}.jar
+guava_jre_url: https://repo1.maven.org/maven2/com/google/guava/guava/{{guava_jre_version}}/guava-{{guava_jre_version}}.jar
+log4j_core_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-core/{{log4j_version}}/log4j-core-{{log4j_version}}.jar
+log4j_api_url: https://repo1.maven.org/maven2/org/apache/logging/log4j/log4j-api/{{log4j_version}}/log4j-api-{{log4j_version}}.jar
+spark_redis_url: https://repo1.maven.org/maven2/com/redislabs/spark-redis_2.12/{{spark_redis_version}}/spark-redis_2.12-{{spark_redis_version}}.jar
+jedis_url: https://repo1.maven.org/maven2/redis/clients/jedis/{{jedis_version}}/jedis-{{jedis_version}}.jar
+zip4j_url: https://repo1.maven.org/maven2/net/lingala/zip4j/zip4j/{{zip4j_version}}/zip4j-{{zip4j_version}}.jar
+guice_url: https://repo1.maven.org/maven2/com/google/inject/guice/{{guice_version}}/guice-{{guice_version}}.jar
+guice_servlet_url: https://repo1.maven.org/maven2/com/google/inject/extensions/guice-servlet/{{guice_version}}/guice-servlet-{{guice_version}}.jar
\ No newline at end of file
diff --git a/ansible/roles/provision-azure-spark-cluster/tasks/main.yml b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml
new file mode 100644
index 0000000000..c126dc8f3b
--- /dev/null
+++ b/ansible/roles/provision-azure-spark-cluster/tasks/main.yml
@@ -0,0 +1,65 @@
+- name: Adding azure blob variable to spark env file
+  lineinfile:
+    path: "{{spark_folder}}/conf/spark-env.sh"
+    line: '{{item.var}}={{item.value}}'
+    regexp: "{{ item.var }}.*"
+  with_items:
+    - {var: 'azure_storage_key', value: '{{ azure_private_storage_account_name }}'}
+    - {var: 'azure_storage_secret', value: '{{ azure_private_storage_account_key }}'}
+  no_log: true
+  when: cloud_service_provider == "azure"
+
+- name: Remove guava-jre, guice default jars
+  become: yes
+  file:
+    path: "{{ spark_folder }}/jars/{{item.var}}-{{item.value}}.jar"
+    state: absent
+  with_items:
+    - {var: 'guava', value: '{{ guava_default_jre_version }}'}
+    - {var: 'guice', value: '{{ guice_default_version }}'}
+    - {var: 'guice-servlet', value: '{{ guice_default_version }}'} 
+   
+- name: Download guava_jre_url and copy to Spark jars folder
+  become: yes
+  get_url: url={{ guava_jre_url }} dest={{ spark_folder }}/jars/guava-{{guava_jre_version}}.jar timeout=1000 force=no
+  
+- name: Download log4j api and copy to Spark jars folder
+  become: yes
+  get_url: url={{ log4j_api_url }} dest={{ spark_folder }}/jars/log4j-api-{{log4j_version}}.jar timeout=1000 force=no
+  
+- name: Download log4j core and copy to Spark jars folder
+  become: yes
+  get_url: url={{ log4j_core_url }} dest={{ spark_folder }}/jars/log4j-core-{{log4j_version}}.jar timeout=1000 force=no
+
+- name: Download spark-redis and copy to Spark jars folder
+  become: yes
+  get_url: url={{ spark_redis_url }} dest={{ spark_folder }}/jars/spark-redis_2.12-{{spark_redis_version}}.jar timeout=1000 force=no
+
+- name: Download jedis and copy to Spark jars folder
+  become: yes
+  get_url: url={{ jedis_url }} dest={{ spark_folder }}/jars/jedis-{{jedis_version}}.jar timeout=1000 force=no
+
+- name: Download zip4j and copy to Spark jars folder
+  become: yes
+  get_url: url={{ zip4j_url }} dest={{ spark_folder }}/jars/zip4j-{{zip4j_version}}.jar timeout=1000 force=no
+
+- name: Download guice and copy to Spark jars folder
+  become: yes
+  get_url: url={{ guice_url }} dest={{ spark_folder }}/jars/guice-{{guice_version}}.jar timeout=1000 force=no
+
+- name: Download guice-servlet and copy to Spark jars folder
+  become: yes
+  get_url: url={{ guice_servlet_url }} dest={{ spark_folder }}/jars/guice-servlet-{{guice_version}}.jar timeout=1000 force=no
+
+- name: Update log4j.properties
+  become: yes
+  blockinfile:
+    path: "{{ spark_folder }}/conf/log4j.properties"
+    block: |
+      log4j.logger.org.ekstep.analytics=INFO
+      log4j.appender.org.ekstep.analytics=org.apache.log4j.RollingFileAppender
+      log4j.appender.org.ekstep.analytics.File=./joblog.log
+      log4j.appender.org.ekstep.analytics.MaxFileSize=${log4jspark.log.maxfilesize}
+      log4j.appender.org.ekstep.analytics.MaxBackupIndex=${log4jspark.log.maxbackupindex}
+      log4j.appender.org.ekstep.analytics.layout=org.apache.log4j.PatternLayout
+      log4j.appender.org.ekstep.analytics.layout.ConversionPattern=%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n
diff --git a/ansible/roles/provision-geo-location-db/defaults/main.yml b/ansible/roles/provision-geo-location-db/defaults/main.yml
index 2b126ea7c4..666fae2ff1 100644
--- a/ansible/roles/provision-geo-location-db/defaults/main.yml
+++ b/ansible/roles/provision-geo-location-db/defaults/main.yml
@@ -1,6 +1,5 @@
-download_user: deployer
-download_group: deployer
-maxmind_geocity_db_url: "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City-CSV.zip"
+download_user: "{{ ansible_ssh_user }}"
+download_group: "{{ ansible_ssh_user }}"
 maxmind_geoip2_csv_converter_url: "https://github.com/maxmind/geoip2-csv-converter/releases/download/v1.0.0/geoip2-csv-converter-v1.0.0-linux-amd64.tar.gz"
 maxmind_db_download_dir: /tmp
 maxmind_db_dir_name: "geoip2-city-csv"
@@ -9,4 +8,4 @@ maxmind_db_unarchived_dir_prefix: "GeoLite2-City-CSV_*"
 maxmind_db_geo_city_blocks_filename: "GeoLite2-City-Blocks-IPv4.csv"
 maxmind_db_geo_city_locations_filename: "GeoLite2-City-Locations-en.csv"
 maxmind_db_geo_city_ip_range_filename: "GeoLite2-City-Range-IPv4.csv"
-maxmind_custom_data_mapping_url: "https://sunbirddevtelemetry.blob.core.windows.net/public/maxmind_custom_data_mapping.csv"
\ No newline at end of file
+maxmind_custom_data_mapping_url: "https://sunbirddevtelemetry.blob.core.windows.net/public/maxmind_custom_data_mapping.csv"
diff --git a/ansible/roles/provision-geo-location-db/files/.gitkeep b/ansible/roles/provision-geo-location-db/files/.gitkeep
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/ansible/roles/provision-geo-location-db/tasks/main.yml b/ansible/roles/provision-geo-location-db/tasks/main.yml
index 74ef8a2497..75ca2b8c06 100644
--- a/ansible/roles/provision-geo-location-db/tasks/main.yml
+++ b/ansible/roles/provision-geo-location-db/tasks/main.yml
@@ -16,8 +16,11 @@
   become: yes
   file: path={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip state=absent
 
-- name: Download Maxmind GeoCity database
-  get_url: url={{ maxmind_geocity_db_url }} dest={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip force=no
+- name: Copy the  Maxmind GeoCity database to remote
+  copy: 
+    src: "files/{{ geoip_zip_csv_file_name }}"
+    dest: "{{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip"
+    force: no
 
 - name: Unarchive Maxmind GeoCity database
   unarchive: src={{ maxmind_db_download_dir }}/{{ maxmind_db_dir_name }}.zip dest={{ maxmind_db_download_dir }} copy=no
@@ -39,9 +42,18 @@
 
 - name: install psycopg2
   become: yes
-  package:
+  apt:
+    state: present
     name: python-psycopg2
+  when: ansible_distribution_version | float < 18
+
+- name: install psycopg2
+  become: yes
+  apt:
     state: present
+    name: python3-psycopg2
+  when: ansible_distribution_version | float > 18
+
 
 - name: Install postgres client to create schema from file
   become: yes
diff --git a/ansible/roles/redis-backup/defaults/main.yml b/ansible/roles/redis-backup/defaults/main.yml
index 155d6ea162..a00387ac74 100644
--- a/ansible/roles/redis-backup/defaults/main.yml
+++ b/ansible/roles/redis-backup/defaults/main.yml
@@ -1,2 +1,4 @@
 redis_backup_dir: /tmp/redis-backup
-redis_backup_azure_container_name: dp-redis-backup
+
+cloud_storage_dpredisbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dpredisbackup_foldername: dp-redis-backup
diff --git a/ansible/roles/redis-backup/meta/main.yml b/ansible/roles/redis-backup/meta/main.yml
deleted file mode 100644
index a124d4f7cb..0000000000
--- a/ansible/roles/redis-backup/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
diff --git a/ansible/roles/redis-backup/tasks/main.yml b/ansible/roles/redis-backup/tasks/main.yml
index 6218852add..aafe94913c 100644
--- a/ansible/roles/redis-backup/tasks/main.yml
+++ b/ansible/roles/redis-backup/tasks/main.yml
@@ -1,14 +1,12 @@
 - name: Create the directory
   file: path={{ redis_backup_dir }} state=directory recurse=yes
   
-
 - set_fact:
     redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}-{{ ansible_hostname }}.zip"
  
 - set_fact:
     redis_backup_file_path: "{{ redis_backup_dir }}/{{ redis_backup_file_name }}"
 
-
 - name: copy dump.rdb file
   copy:
     src: /home/analytics/redis-stable/dump.rdb
@@ -19,15 +17,44 @@
   archive: path="{{ redis_backup_dir }}/*" dest="{{ redis_backup_dir }}/{{ redis_backup_file_name }}" format=zip
   async: 3600
   poll: 10
-    
 
-- name: upload to azure
+- name: upload file to azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-upload.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dpredisbackup_foldername }}"
+    container_public_access: "off"
+    blob_file_name: "{{ redis_backup_file_name }}"
+    local_file_or_folder_path: "{{ redis_backup_file_path }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+
+- name: upload file to aws s3
   include_role:
-    name: artifacts-upload-azure
+    name: aws-cloud-storage
+    tasks_from: upload.yml
   vars:
-    artifact: "{{ redis_backup_file_name }}"
-    artifact_path: "{{ redis_backup_file_path }}"
-    artifacts_container: "{{ redis_backup_azure_container_name }}"
+    s3_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "{{ redis_backup_file_path }}"
+    s3_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_backup_file_name }}"
+  when: cloud_service_provider == "aws"
    
+- name: upload file to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_backup_file_name }}"
+    local_file_or_folder_path: "{{ redis_backup_file_path }}"
+  when: cloud_service_provider == "gcloud"
+
 - name: clean up backup dir after upload
   file: path={{ redis_backup_dir }} state=absent
diff --git a/ansible/roles/redis-multiprocess-backup/defaults/main.yml b/ansible/roles/redis-multiprocess-backup/defaults/main.yml
new file mode 100644
index 0000000000..849e3099bf
--- /dev/null
+++ b/ansible/roles/redis-multiprocess-backup/defaults/main.yml
@@ -0,0 +1,15 @@
+redis_backup_dir: /tmp/redis-backup
+redis_data_dir: /data
+redis:
+  config:
+    device:
+      name: 'device'
+    user:
+      name: 'user'
+    content:
+      name: 'content'
+    dialcode:
+      name: 'dialcode'
+
+cloud_storage_dpredisbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dpredisbackup_foldername: dp-redis-backup
diff --git a/ansible/roles/redis-multiprocess-backup/tasks/main.yml b/ansible/roles/redis-multiprocess-backup/tasks/main.yml
new file mode 100644
index 0000000000..7e31c94adb
--- /dev/null
+++ b/ansible/roles/redis-multiprocess-backup/tasks/main.yml
@@ -0,0 +1,61 @@
+- name: Create the directory
+  file: path={{ redis_backup_dir }}/{{ item.value.name }} state=directory recurse=yes
+  with_dict: "{{ redis.config }}"
+
+- set_fact:
+    redis_backup_file_name: "redis-backup-{{ lookup('pipe', 'date +%Y-%m-%d-%T') }}-{{ ansible_hostname }}.zip"
+ 
+- name: copy dump.rdb files
+  copy:
+    src: "{{ redis_data_dir }}/{{ item.value.name }}-dump.rdb"
+    dest: "{{ redis_backup_dir }}/{{ item.value.name }}/{{ item.value.name }}-dump.rdb"
+    remote_src: yes
+  with_dict: "{{ redis.config }}"
+
+- name: Create archive of backup directory
+  archive: path="{{ redis_backup_dir }}/{{ item.value.name }}/*" dest="{{ redis_backup_dir }}/{{ item.value.name }}/{{ redis_backup_file_name }}" format=zip remove=yes
+  async: 3600
+  poll: 10
+  with_dict: "{{ redis.config }}"
+
+- name: upload file to azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-upload.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dpredisbackup_foldername }}"
+    container_public_access: "off"
+    blob_file_name: "{{ item.value.name }}/{{ redis_backup_file_name }}"
+    local_file_or_folder_path: "{{ redis_backup_dir }}/{{ item.value.name }}/{{ redis_backup_file_name }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  with_dict: "{{ redis.config }}"  
+  when: cloud_service_provider == "azure"
+
+- name: upload backup to S3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: upload-folder.yml
+  vars:
+    local_file_or_folder_path: "{{ redis_backup_dir }}/*"
+    s3_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    s3_path: "{{ cloud_storage_dpredisbackup_foldername }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "aws"
+   
+- name: upload files to gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: upload-batch.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}"
+    local_file_or_folder_path: "{{ redis_backup_dir }}/*"
+  when: cloud_service_provider == "gcloud"
+
+- name: clean up backup dir after upload
+  file: path={{ redis_backup_dir }} state=absent
diff --git a/ansible/roles/redis-multiprocess-restore/defaults/main.yml b/ansible/roles/redis-multiprocess-restore/defaults/main.yml
new file mode 100644
index 0000000000..84f07feb16
--- /dev/null
+++ b/ansible/roles/redis-multiprocess-restore/defaults/main.yml
@@ -0,0 +1,11 @@
+redis_user: analytics
+redis_restore_user: analytics
+analytics_user_home: /home/analytics
+
+device_port: "{{ device_port }}"
+user_port: "{{ user_port }}"
+content_port: "{{ content_port }}"
+dialcode_port: "{{ dialcode_port }}"
+
+cloud_storage_dpredisbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dpredisbackup_foldername: dp-redis-backup
diff --git a/ansible/roles/redis-multiprocess-restore/tasks/main.yml b/ansible/roles/redis-multiprocess-restore/tasks/main.yml
new file mode 100644
index 0000000000..87f1838a19
--- /dev/null
+++ b/ansible/roles/redis-multiprocess-restore/tasks/main.yml
@@ -0,0 +1,87 @@
+---
+
+- name: Create the directory
+  become: yes
+  become_user: "{{ redis_restore_user }}"
+  file: path=/tmp/{{ item }} state=directory recurse=yes
+  with_items: "{{ redis_restore_process }}"
+
+- name: download a file from azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dpredisbackup_foldername }}"
+    blob_file_name: "{{ item }}/{{ redis_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ item }}/{{ redis_restore_file_name }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  with_items: "{{ redis_restore_process }}"
+  when: cloud_service_provider == "azure"
+
+- name: download a file from aws s3
+  become: true
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "/tmp/{{ item }}/{{ redis_restore_file_name }}"
+    s3_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ item }}/{{ redis_restore_file_name }}"
+  when: cloud_service_provider == "aws"  
+
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ item }}/{{ redis_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ item }}/{{ redis_restore_file_name }}"
+  with_items: "{{ redis_restore_process }}"
+  when: cloud_service_provider == "gcloud"
+
+- name: get redis process ports to stop 
+  shell: "echo {{ lookup ('vars', item + '_port') }}"
+  register: ports
+  with_items: "{{ redis_restore_process }}" 
+
+- name: stop redis to restore backup 
+  become: yes
+  become_user: "{{ redis_restore_user }}"
+  shell: "echo shutdown | {{ analytics_user_home }}/redis-stable/src/redis-cli -p {{ item.stdout }}"
+  with_items: "{{ ports.results }}"
+
+- name: Extract backup zip into tmp
+  become: yes
+  become_user: "{{ redis_restore_user }}"
+  unarchive:
+    src: "/tmp/{{ item }}/{{ redis_restore_file_name }}"
+    dest: "/tmp/{{ item }}"
+    remote_src: True
+  with_items: "{{ redis_restore_process }}"  
+
+- name: copy dump.rdb files to data dir
+  become: yes
+  become_user: "{{ redis_restore_user }}"
+  shell: cp /tmp/{{ item }}/{{ item }}-dump.rdb {{ analytics_user_home }}/redis-stable/{{ item }}-dump.rdb
+  with_items: "{{ redis_restore_process }}"
+
+- name: start redis
+  become: yes
+  become_user: "{{ redis_restore_user }}"
+  command: '{{ analytics_user_home }}/redis-stable/src/redis-server {{ analytics_user_home }}/redis-stable/redis-{{ item }}.conf'
+  with_items: "{{ redis_restore_process }}"
+
+- name: Keyspace info
+  shell: "echo info keyspace | {{ analytics_user_home }}/redis-stable/src/redis-cli -p {{ item.stdout }}"
+  register: restoreinfo
+  with_items: "{{ ports.results }}"
+
+- debug: var=item.stdout_lines
+  with_items: "{{ restoreinfo.results }}"  
diff --git a/ansible/roles/redis-multiprocess/defaults/main.yml b/ansible/roles/redis-multiprocess/defaults/main.yml
new file mode 100644
index 0000000000..404aaa56d7
--- /dev/null
+++ b/ansible/roles/redis-multiprocess/defaults/main.yml
@@ -0,0 +1,28 @@
+redis_user: analytics
+analytics_redis_user_home: /home/{{ redis_user }}
+redis_home: "{{ analytics_redis_user_home }}/redis/redis-{{ redis_version }}"
+redis_tar_ball: "http://download.redis.io/releases/redis-{{ redis_version }}.tar.gz"
+redis_version: 6.2.5
+redis_dir: "{{ analytics_redis_user_home }}/redis"
+redis_dump_rdb_time_in_seconds: 900
+redis_dump_rdb_keys_to_save_state: 100
+redis_data_dir: /data
+
+redis:
+  config:
+    device:
+      port: "{{ device_port }}"
+      name: 'device'
+      max_memory: 100mb
+    user:
+      port: "{{ user_port }}"
+      name: 'user'
+      max_memory: 100mb
+    content:
+      port: "{{ content_port }}"
+      name: 'content'
+      max_memory: 100mb
+    dialcode:
+      port: "{{ dialcode_port }}"
+      name: 'dialcode'
+      max_memory: 100mb     
diff --git a/ansible/roles/redis-multiprocess/tasks/main.yml b/ansible/roles/redis-multiprocess/tasks/main.yml
new file mode 100755
index 0000000000..7d1d13c1a3
--- /dev/null
+++ b/ansible/roles/redis-multiprocess/tasks/main.yml
@@ -0,0 +1,60 @@
+---
+# tasks file for redis
+- name: change the data dir ownership
+  file:
+    path: /data
+    state: directory
+    owner: "{{ redis_user }}"
+    group: "{{ redis_user }}"
+    mode: 0755
+  become: yes
+
+- name: create the redis dir
+  become: yes
+  become_user: "{{ redis_user }}"
+  file:
+    path: "{{ analytics_redis_user_home }}/redis"
+    state: directory
+
+- name: download redis zipfile
+  become: yes
+  become_user: "{{ redis_user }}"
+  get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-{{ redis_version }}.tar.gz timeout=50 force=no owner={{ redis_user }}
+
+- name: unzip
+  become: yes
+  become_user: "{{ redis_user }}"
+  unarchive: src={{ redis_dir }}/redis-{{ redis_version }}.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }}
+
+- name: Change ownership of redis installation
+  become: yes
+  file: path={{ redis_home }} owner={{ redis_user }} group={{ redis_user }} state=directory recurse=yes
+
+- name: Copy redis conf files
+  become: yes
+  become_user: "{{ redis_user }}"
+  template: src=redis.conf.j2 dest={{ redis_dir }}/redis-{{ redis_version }}/redis-{{ item.value.name }}.conf
+  vars:
+    _port_: "{{ item.value.port }}"
+    _name_: "{{ item.value.name }}"
+    max_memory: "{{ item.value.max_memory }}"
+  with_dict: "{{ redis.config }}"
+
+- name: Install required utils
+  apt: name={{ item }} state=present
+  become: yes
+  with_items:
+    - make
+    - gcc
+    - build-essential
+
+- name: Go to the folder and run make
+  become: yes
+  become_user: "{{ redis_user }}"
+  command: chdir={{ redis_home }} make
+
+- name: start redis
+  become: yes
+  become_user: "{{ redis_user }}"
+  command: '{{ redis_dir }}/redis-{{ redis_version }}/src/redis-server {{ redis_dir }}/redis-{{ redis_version }}/redis-{{ item.value.name }}.conf'
+  with_dict: "{{ redis.config }}" 
diff --git a/ansible/roles/redis-multiprocess/templates/redis.conf.j2 b/ansible/roles/redis-multiprocess/templates/redis.conf.j2
new file mode 100644
index 0000000000..456cc0d42f
--- /dev/null
+++ b/ansible/roles/redis-multiprocess/templates/redis.conf.j2
@@ -0,0 +1,1026 @@
+# Redis configuration file example.
+#
+# Note that in order to read the configuration file, Redis must be
+# started with the file path as first argument:
+#
+# ./redis-server /path/to/redis.conf
+
+# Note on units: when memory size is needed, it is possible to specify
+# it in the usual form of 1k 5GB 4M and so forth:
+#
+# 1k => 1000 bytes
+# 1kb => 1024 bytes
+# 1m => 1000000 bytes
+# 1mb => 1024*1024 bytes
+# 1g => 1000000000 bytes
+# 1gb => 1024*1024*1024 bytes
+#
+# units are case insensitive so 1GB 1Gb 1gB are all the same.
+
+################################## INCLUDES ###################################
+
+# Include one or more other config files here.  This is useful if you
+# have a standard template that goes to all Redis servers but also need
+# to customize a few per-server settings.  Include files can include
+# other files, so use this wisely.
+#
+# Notice option "include" won't be rewritten by command "CONFIG REWRITE"
+# from admin or Redis Sentinel. Since Redis always uses the last processed
+# line as value of a configuration directive, you'd better put includes
+# at the beginning of this file to avoid overwriting config change at runtime.
+#
+# If instead you are interested in using includes to override configuration
+# options, it is better to use include as the last line.
+#
+# include /path/to/local.conf
+# include /path/to/other.conf
+
+################################## NETWORK #####################################
+
+# By default, if no "bind" configuration directive is specified, Redis listens
+# for connections from all the network interfaces available on the server.
+# It is possible to listen to just one or multiple selected interfaces using
+# the "bind" configuration directive, followed by one or more IP addresses.
+#
+# Examples:
+#
+# bind 192.168.1.100 10.0.0.1
+# bind 127.0.0.1 ::1
+#
+# ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the
+# internet, binding to all the interfaces is dangerous and will expose the
+# instance to everybody on the internet. So by default we uncomment the
+# following bind directive, that will force Redis to listen only into
+# the IPv4 lookback interface address (this means Redis will be able to
+# accept connections only from clients running into the same computer it
+# is running).
+#
+# IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES
+# JUST COMMENT THE FOLLOWING LINE.
+# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+# bind 127.0.0.1
+
+# Protected mode is a layer of security protection, in order to avoid that
+# Redis instances left open on the internet are accessed and exploited.
+#
+# When protected mode is on and if:
+#
+# 1) The server is not binding explicitly to a set of addresses using the
+#    "bind" directive.
+# 2) No password is configured.
+#
+# The server only accepts connections from clients connecting from the
+# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain
+# sockets.
+#
+# By default protected mode is enabled. You should disable it only if
+# you are sure you want clients from other hosts to connect to Redis
+# even if no authentication is configured, nor a specific set of interfaces
+# are explicitly listed using the "bind" directive.
+protected-mode no
+
+# Accept connections on the specified port, default is {{ _port_ }} (IANA #815344).
+# If port 0 is specified Redis will not listen on a TCP socket.
+port {{ _port_ }}
+
+# TCP listen() backlog.
+#
+# In high requests-per-second environments you need an high backlog in order
+# to avoid slow clients connections issues. Note that the Linux kernel
+# will silently truncate it to the value of /proc/sys/net/core/somaxconn so
+# make sure to raise both the value of somaxconn and tcp_max_syn_backlog
+# in order to get the desired effect.
+tcp-backlog 511
+
+# Unix socket.
+#
+# Specify the path for the Unix socket that will be used to listen for
+# incoming connections. There is no default, so Redis will not listen
+# on a unix socket when not specified.
+#
+# unixsocket /tmp/redis.sock
+# unixsocketperm 700
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 0
+
+# TCP keepalive.
+#
+# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence
+# of communication. This is useful for two reasons:
+#
+# 1) Detect dead peers.
+# 2) Take the connection alive from the point of view of network
+#    equipment in the middle.
+#
+# On Linux, the specified value (in seconds) is the period used to send ACKs.
+# Note that to close the connection the double of the time is needed.
+# On other kernels the period depends on the kernel configuration.
+#
+# A reasonable value for this option is 300 seconds, which is the new
+# Redis default starting with Redis 3.2.1.
+tcp-keepalive 300
+
+################################# GENERAL #####################################
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# If you run Redis from upstart or systemd, Redis can interact with your
+# supervision tree. Options:
+#   supervised no      - no supervision interaction
+#   supervised upstart - signal upstart by putting Redis into SIGSTOP mode
+#   supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET
+#   supervised auto    - detect upstart or systemd method based on
+#                        UPSTART_JOB or NOTIFY_SOCKET environment variables
+# Note: these supervision methods only signal "process is ready."
+#       They do not enable continuous liveness pings back to your supervisor.
+supervised systemd
+
+# If a pid file is specified, Redis writes it where specified at startup
+# and removes it at exit.
+#
+# When the server runs non daemonized, no pid file is created if none is
+# specified in the configuration. When the server is daemonized, the pid file
+# is used even if not specified, defaulting to "/var/run/redis.pid".
+#
+# Creating a pid file is best effort: if Redis is not able to create it
+# nothing bad happens, the server will start and run normally.
+pidfile /var/run/redis_{{ _name_ }}.pid
+
+# Specify the server verbosity level.
+# This can be one of:
+# debug (a lot of information, useful for development/testing)
+# verbose (many rarely useful info, but not a mess like the debug level)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel notice
+
+# Specify the log file name. Also the empty string can be used to force
+# Redis to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile "{{ redis_data_dir }}/redis-{{ _name_ }}.log"
+
+
+# To enable logging to the system logger, just set 'syslog-enabled' to yes,
+# and optionally update the other syslog parameters to suit your needs.
+# syslog-enabled no
+
+# Specify the syslog identity.
+# syslog-ident redis
+
+# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7.
+# syslog-facility local0
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################ SNAPSHOTTING  ################################
+#
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+#
+#   Note: you can disable saving completely by commenting out all "save" lines.
+#
+#   It is also possible to remove all the previously configured save
+#   points by adding a save directive with a single empty string argument
+#   like in the following example:
+#
+#   save ""
+
+#save 900 1
+#save 300 10
+#save 60 10000
+save {{ redis_dump_rdb_time_in_seconds }} {{ redis_dump_rdb_keys_to_save_state }}
+
+# By default Redis will stop accepting writes if RDB snapshots are enabled
+# (at least one save point) and the latest background save failed.
+# This will make the user aware (in a hard way) that data is not persisting
+# on disk properly, otherwise chances are that no one will notice and some
+# disaster will happen.
+#
+# If the background saving process will start working again Redis will
+# automatically allow writes again.
+#
+# However if you have setup your proper monitoring of the Redis server
+# and persistence, you may want to disable this feature so that Redis will
+# continue to work as usual even if there are problems with disk,
+# permissions, and so forth.
+stop-writes-on-bgsave-error yes
+
+# Compress string objects using LZF when dump .rdb databases?
+# For default that's set to 'yes' as it's almost always a win.
+# If you want to save some CPU in the saving child set it to 'no' but
+# the dataset will likely be bigger if you have compressible values or keys.
+rdbcompression yes
+
+# Since version 5 of RDB a CRC64 checksum is placed at the end of the file.
+# This makes the format more resistant to corruption but there is a performance
+# hit to pay (around 10%) when saving and loading RDB files, so you can disable it
+# for maximum performances.
+#
+# RDB files created with checksum disabled have a checksum of zero that will
+# tell the loading code to skip the check.
+rdbchecksum yes
+
+# The filename where to dump the DB
+dbfilename {{ _name_ }}-dump.rdb
+
+# The working directory.
+#
+# The DB will be written inside this directory, with the filename specified
+# above using the 'dbfilename' configuration directive.
+#
+# The Append Only File will also be created inside this directory.
+#
+# Note that you must specify a directory here, not a file name.
+dir "{{redis_data_dir}}"
+
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. A few things to understand ASAP about Redis replication.
+#
+# 1) Redis replication is asynchronous, but you can configure a master to
+#    stop accepting writes if it appears to be not connected with at least
+#    a given number of slaves.
+# 2) Redis slaves are able to perform a partial resynchronization with the
+#    master if the replication link is lost for a relatively small amount of
+#    time. You may want to configure the replication backlog size (see the next
+#    sections of this file) with a sensible value depending on your needs.
+# 3) Replication is automatic and does not need user intervention. After a
+#    network partition slaves automatically try to reconnect to masters
+#    and resynchronize with them.
+#
+# slaveof <masterip> <masterport>
+
+# If the master is password protected (using the "requirepass" configuration
+# directive below) it is possible to tell the slave to authenticate before
+# starting the replication synchronization process, otherwise the master will
+# refuse the slave request.
+#
+# masterauth <master-password>
+
+# When a slave loses its connection with the master, or when the replication
+# is still in progress, the slave can act in two different ways:
+#
+# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
+#    still reply to client requests, possibly with out of date data, or the
+#    data set may just be empty if this is the first synchronization.
+#
+# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
+#    an error "SYNC with master in progress" to all the kind of commands
+#    but to INFO and SLAVEOF.
+#
+slave-serve-stale-data yes
+
+# You can configure a slave instance to accept writes or not. Writing against
+# a slave instance may be useful to store some ephemeral data (because data
+# written on a slave will be easily deleted after resync with the master) but
+# may also cause problems if clients are writing to it because of a
+# misconfiguration.
+#
+# Since Redis 2.6 by default slaves are read-only.
+#
+# Note: read only slaves are not designed to be exposed to untrusted clients
+# on the internet. It's just a protection layer against misuse of the instance.
+# Still a read only slave exports by default all the administrative commands
+# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve
+# security of read only slaves using 'rename-command' to shadow all the
+# administrative / dangerous commands.
+slave-read-only yes
+
+# Replication SYNC strategy: disk or socket.
+#
+# -------------------------------------------------------
+# WARNING: DISKLESS REPLICATION IS EXPERIMENTAL CURRENTLY
+# -------------------------------------------------------
+#
+# New slaves and reconnecting slaves that are not able to continue the replication
+# process just receiving differences, need to do what is called a "full
+# synchronization". An RDB file is transmitted from the master to the slaves.
+# The transmission can happen in two different ways:
+#
+# 1) Disk-backed: The Redis master creates a new process that writes the RDB
+#                 file on disk. Later the file is transferred by the parent
+#                 process to the slaves incrementally.
+# 2) Diskless: The Redis master creates a new process that directly writes the
+#              RDB file to slave sockets, without touching the disk at all.
+#
+# With disk-backed replication, while the RDB file is generated, more slaves
+# can be queued and served with the RDB file as soon as the current child producing
+# the RDB file finishes its work. With diskless replication instead once
+# the transfer starts, new slaves arriving will be queued and a new transfer
+# will start when the current one terminates.
+#
+# When diskless replication is used, the master waits a configurable amount of
+# time (in seconds) before starting the transfer in the hope that multiple slaves
+# will arrive and the transfer can be parallelized.
+#
+# With slow disks and fast (large bandwidth) networks, diskless replication
+# works better.
+repl-diskless-sync no
+
+# When diskless replication is enabled, it is possible to configure the delay
+# the server waits in order to spawn the child that transfers the RDB via socket
+# to the slaves.
+#
+# This is important since once the transfer starts, it is not possible to serve
+# new slaves arriving, that will be queued for the next RDB transfer, so the server
+# waits a delay in order to let more slaves arrive.
+#
+# The delay is specified in seconds, and by default is 5 seconds. To disable
+# it entirely just set it to 0 seconds and the transfer will start ASAP.
+repl-diskless-sync-delay 5
+
+# Slaves send PINGs to server in a predefined interval. It's possible to change
+# this interval with the repl_ping_slave_period option. The default value is 10
+# seconds.
+#
+# repl-ping-slave-period 10
+
+# The following option sets the replication timeout for:
+#
+# 1) Bulk transfer I/O during SYNC, from the point of view of slave.
+# 2) Master timeout from the point of view of slaves (data, pings).
+# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings).
+#
+# It is important to make sure that this value is greater than the value
+# specified for repl-ping-slave-period otherwise a timeout will be detected
+# every time there is low traffic between the master and the slave.
+#
+# repl-timeout 60
+
+# Disable TCP_NODELAY on the slave socket after SYNC?
+#
+# If you select "yes" Redis will use a smaller number of TCP packets and
+# less bandwidth to send data to slaves. But this can add a delay for
+# the data to appear on the slave side, up to 40 milliseconds with
+# Linux kernels using a default configuration.
+#
+# If you select "no" the delay for data to appear on the slave side will
+# be reduced but more bandwidth will be used for replication.
+#
+# By default we optimize for low latency, but in very high traffic conditions
+# or when the master and slaves are many hops away, turning this to "yes" may
+# be a good idea.
+repl-disable-tcp-nodelay no
+
+# Set the replication backlog size. The backlog is a buffer that accumulates
+# slave data when slaves are disconnected for some time, so that when a slave
+# wants to reconnect again, often a full resync is not needed, but a partial
+# resync is enough, just passing the portion of data the slave missed while
+# disconnected.
+#
+# The bigger the replication backlog, the longer the time the slave can be
+# disconnected and later be able to perform a partial resynchronization.
+#
+# The backlog is only allocated once there is at least a slave connected.
+#
+# repl-backlog-size 1mb
+
+# After a master has no longer connected slaves for some time, the backlog
+# will be freed. The following option configures the amount of seconds that
+# need to elapse, starting from the time the last slave disconnected, for
+# the backlog buffer to be freed.
+#
+# A value of 0 means to never release the backlog.
+#
+# repl-backlog-ttl 3600
+
+# The slave priority is an integer number published by Redis in the INFO output.
+# It is used by Redis Sentinel in order to select a slave to promote into a
+# master if the master is no longer working correctly.
+#
+# A slave with a low priority number is considered better for promotion, so
+# for instance if there are three slaves with priority 10, 100, 25 Sentinel will
+# pick the one with priority 10, that is the lowest.
+#
+# However a special priority of 0 marks the slave as not able to perform the
+# role of master, so a slave with priority of 0 will never be selected by
+# Redis Sentinel for promotion.
+#
+# By default the priority is 100.
+slave-priority 100
+
+# It is possible for a master to stop accepting writes if there are less than
+# N slaves connected, having a lag less or equal than M seconds.
+#
+# The N slaves need to be in "online" state.
+#
+# The lag in seconds, that must be <= the specified value, is calculated from
+# the last ping received from the slave, that is usually sent every second.
+#
+# This option does not GUARANTEE that N replicas will accept the write, but
+# will limit the window of exposure for lost writes in case not enough slaves
+# are available, to the specified number of seconds.
+#
+# For example to require at least 3 slaves with a lag <= 10 seconds use:
+#
+# min-slaves-to-write 3
+# min-slaves-max-lag 10
+#
+# Setting one or the other to 0 disables the feature.
+#
+# By default min-slaves-to-write is set to 0 (feature disabled) and
+# min-slaves-max-lag is set to 10.
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+#
+# Warning: since Redis is pretty fast an outside user can try up to
+# 150k passwords per second against a good box. This means that you should
+# use a very strong password otherwise it will be very easy to break.
+#
+# requirepass foobared
+
+# Command renaming.
+#
+# It is possible to change the name of dangerous commands in a shared
+# environment. For instance the CONFIG command may be renamed into something
+# hard to guess so that it will still be available for internal-use tools
+# but not available for general clients.
+#
+# Example:
+#
+# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52
+#
+# It is also possible to completely kill a command by renaming it into
+# an empty string:
+#
+# rename-command CONFIG ""
+#
+# Please note that changing the name of commands that are logged into the
+# AOF file or transmitted to slaves may cause problems.
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default
+# this limit is set to 10000 clients, however if the Redis server is not
+# able to configure the process file limit to allow for the specified limit
+# the max number of allowed clients is set to the current file limit
+# minus 32 (as Redis reserves a few file descriptors for internal uses).
+#
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+#
+# maxclients 10000
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys
+# according to the eviction policy selected (see maxmemory-policy).
+#
+# If Redis can't remove keys according to the policy, or if the policy is
+# set to 'noeviction', Redis will start to reply with errors to commands
+# that would use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to read-only commands like GET.
+#
+# This option is usually useful when using Redis as an LRU cache, or to set
+# a hard memory limit for an instance (using the 'noeviction' policy).
+#
+# WARNING: If you have slaves attached to an instance with maxmemory on,
+# the size of the output buffers needed to feed the slaves are subtracted
+# from the used memory count, so that network problems / resyncs will
+# not trigger a loop where keys are evicted, and in turn the output
+# buffer of slaves is full with DELs of keys evicted triggering the deletion
+# of more keys, and so forth until the database is completely emptied.
+#
+# In short... if you have slaves attached it is suggested that you set a lower
+# limit for maxmemory so that there is some free RAM on the system for slave
+# output buffers (but this is not needed if the policy is 'noeviction').
+#
+maxmemory {{ max_memory }}
+
+# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory
+# is reached. You can select among five behaviors:
+#
+# volatile-lru -> remove the key with an expire set using an LRU algorithm
+# allkeys-lru -> remove any key according to the LRU algorithm
+# volatile-random -> remove a random key with an expire set
+# allkeys-random -> remove a random key, any key
+# volatile-ttl -> remove the key with the nearest expire time (minor TTL)
+# noeviction -> don't expire at all, just return an error on write operations
+#
+# Note: with any of the above policies, Redis will return an error on write
+#       operations, when there are no suitable keys for eviction.
+#
+#       At the date of writing these commands are: set setnx setex append
+#       incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd
+#       sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby
+#       zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby
+#       getset mset msetnx exec sort
+#
+# The default is:
+#
+# maxmemory-policy noeviction
+
+# LRU and minimal TTL algorithms are not precise algorithms but approximated
+# algorithms (in order to save memory), so you can tune it for speed or
+# accuracy. For default Redis will check five keys and pick the one that was
+# used less recently, you can change the sample size using the following
+# configuration directive.
+#
+# The default of 5 produces good enough results. 10 Approximates very closely
+# true LRU but costs a bit more CPU. 3 is very fast but not very accurate.
+#
+# maxmemory-samples 5
+
+############################## APPEND ONLY MODE ###############################
+
+# By default Redis asynchronously dumps the dataset on disk. This mode is
+# good enough in many applications, but an issue with the Redis process or
+# a power outage may result into a few minutes of writes lost (depending on
+# the configured save points).
+#
+# The Append Only File is an alternative persistence mode that provides
+# much better durability. For instance using the default data fsync policy
+# (see later in the config file) Redis can lose just one second of writes in a
+# dramatic event like a server power outage, or a single write if something
+# wrong with the Redis process itself happens, but the operating system is
+# still running correctly.
+#
+# AOF and RDB persistence can be enabled at the same time without problems.
+# If the AOF is enabled on startup Redis will load the AOF, that is the file
+# with the better durability guarantees.
+#
+# Please check http://redis.io/topics/persistence for more information.
+
+appendonly no
+
+# The name of the append only file (default: "appendonly.aof")
+
+appendfilename "appendonly.aof"
+
+# The fsync() call tells the Operating System to actually write data on disk
+# instead of waiting for more data in the output buffer. Some OS will really flush
+# data on disk, some other OS will just try to do it ASAP.
+#
+# Redis supports three different modes:
+#
+# no: don't fsync, just let the OS flush the data when it wants. Faster.
+# always: fsync after every write to the append only log. Slow, Safest.
+# everysec: fsync only one time every second. Compromise.
+#
+# The default is "everysec", as that's usually the right compromise between
+# speed and data safety. It's up to you to understand if you can relax this to
+# "no" that will let the operating system flush the output buffer when
+# it wants, for better performances (but if you can live with the idea of
+# some data loss consider the default persistence mode that's snapshotting),
+# or on the contrary, use "always" that's very slow but a bit safer than
+# everysec.
+#
+# More details please check the following article:
+# http://antirez.com/post/redis-persistence-demystified.html
+#
+# If unsure, use "everysec".
+
+# appendfsync always
+appendfsync everysec
+# appendfsync no
+
+# When the AOF fsync policy is set to always or everysec, and a background
+# saving process (a background save or AOF log background rewriting) is
+# performing a lot of I/O against the disk, in some Linux configurations
+# Redis may block too long on the fsync() call. Note that there is no fix for
+# this currently, as even performing fsync in a different thread will block
+# our synchronous write(2) call.
+#
+# In order to mitigate this problem it's possible to use the following option
+# that will prevent fsync() from being called in the main process while a
+# BGSAVE or BGREWRITEAOF is in progress.
+#
+# This means that while another child is saving, the durability of Redis is
+# the same as "appendfsync none". In practical terms, this means that it is
+# possible to lose up to 30 seconds of log in the worst scenario (with the
+# default Linux settings).
+#
+# If you have latency problems turn this to "yes". Otherwise leave it as
+# "no" that is the safest pick from the point of view of durability.
+
+no-appendfsync-on-rewrite no
+
+# Automatic rewrite of the append only file.
+# Redis is able to automatically rewrite the log file implicitly calling
+# BGREWRITEAOF when the AOF log size grows by the specified percentage.
+#
+# This is how it works: Redis remembers the size of the AOF file after the
+# latest rewrite (if no rewrite has happened since the restart, the size of
+# the AOF at startup is used).
+#
+# This base size is compared to the current size. If the current size is
+# bigger than the specified percentage, the rewrite is triggered. Also
+# you need to specify a minimal size for the AOF file to be rewritten, this
+# is useful to avoid rewriting the AOF file even if the percentage increase
+# is reached but it is still pretty small.
+#
+# Specify a percentage of zero in order to disable the automatic AOF
+# rewrite feature.
+
+auto-aof-rewrite-percentage 100
+auto-aof-rewrite-min-size 64mb
+
+# An AOF file may be found to be truncated at the end during the Redis
+# startup process, when the AOF data gets loaded back into memory.
+# This may happen when the system where Redis is running
+# crashes, especially when an ext4 filesystem is mounted without the
+# data=ordered option (however this can't happen when Redis itself
+# crashes or aborts but the operating system still works correctly).
+#
+# Redis can either exit with an error when this happens, or load as much
+# data as possible (the default now) and start if the AOF file is found
+# to be truncated at the end. The following option controls this behavior.
+#
+# If aof-load-truncated is set to yes, a truncated AOF file is loaded and
+# the Redis server starts emitting a log to inform the user of the event.
+# Otherwise if the option is set to no, the server aborts with an error
+# and refuses to start. When the option is set to no, the user requires
+# to fix the AOF file using the "redis-check-aof" utility before to restart
+# the server.
+#
+# Note that if the AOF file will be found to be corrupted in the middle
+# the server will still exit with an error. This option only applies when
+# Redis will try to read more data from the AOF file but not enough bytes
+# will be found.
+aof-load-truncated yes
+
+################################ LUA SCRIPTING  ###############################
+
+# Max execution time of a Lua script in milliseconds.
+#
+# If the maximum execution time is reached Redis will log that a script is
+# still in execution after the maximum allowed time and will start to
+# reply to queries with an error.
+#
+# When a long running script exceeds the maximum execution time only the
+# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be
+# used to stop a script that did not yet called write commands. The second
+# is the only way to shut down the server in the case a write command was
+# already issued by the script but the user doesn't want to wait for the natural
+# termination of the script.
+#
+# Set it to 0 or a negative value for unlimited execution without warnings.
+lua-time-limit 5000
+
+################################ REDIS CLUSTER  ###############################
+#
+# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+# WARNING EXPERIMENTAL: Redis Cluster is considered to be stable code, however
+# in order to mark it as "mature" we need to wait for a non trivial percentage
+# of users to deploy it in production.
+# ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
+#
+# Normal Redis instances can't be part of a Redis Cluster; only nodes that are
+# started as cluster nodes can. In order to start a Redis instance as a
+# cluster node enable the cluster support uncommenting the following:
+#
+# cluster-enabled yes
+
+# Every cluster node has a cluster configuration file. This file is not
+# intended to be edited by hand. It is created and updated by Redis nodes.
+# Every Redis Cluster node requires a different cluster configuration file.
+# Make sure that instances running in the same system do not have
+# overlapping cluster configuration file names.
+#
+# cluster-config-file nodes-{{ _name_ }}.conf
+
+# Cluster node timeout is the amount of milliseconds a node must be unreachable
+# for it to be considered in failure state.
+# Most other internal time limits are multiple of the node timeout.
+#
+# cluster-node-timeout 15000
+
+# A slave of a failing master will avoid to start a failover if its data
+# looks too old.
+#
+# There is no simple way for a slave to actually have a exact measure of
+# its "data age", so the following two checks are performed:
+#
+# 1) If there are multiple slaves able to failover, they exchange messages
+#    in order to try to give an advantage to the slave with the best
+#    replication offset (more data from the master processed).
+#    Slaves will try to get their rank by offset, and apply to the start
+#    of the failover a delay proportional to their rank.
+#
+# 2) Every single slave computes the time of the last interaction with
+#    its master. This can be the last ping or command received (if the master
+#    is still in the "connected" state), or the time that elapsed since the
+#    disconnection with the master (if the replication link is currently down).
+#    If the last interaction is too old, the slave will not try to failover
+#    at all.
+#
+# The point "2" can be tuned by user. Specifically a slave will not perform
+# the failover if, since the last interaction with the master, the time
+# elapsed is greater than:
+#
+#   (node-timeout * slave-validity-factor) + repl-ping-slave-period
+#
+# So for example if node-timeout is 30 seconds, and the slave-validity-factor
+# is 10, and assuming a default repl-ping-slave-period of 10 seconds, the
+# slave will not try to failover if it was not able to talk with the master
+# for longer than 310 seconds.
+#
+# A large slave-validity-factor may allow slaves with too old data to failover
+# a master, while a too small value may prevent the cluster from being able to
+# elect a slave at all.
+#
+# For maximum availability, it is possible to set the slave-validity-factor
+# to a value of 0, which means, that slaves will always try to failover the
+# master regardless of the last time they interacted with the master.
+# (However they'll always try to apply a delay proportional to their
+# offset rank).
+#
+# Zero is the only value able to guarantee that when all the partitions heal
+# the cluster will always be able to continue.
+#
+# cluster-slave-validity-factor 10
+
+# Cluster slaves are able to migrate to orphaned masters, that are masters
+# that are left without working slaves. This improves the cluster ability
+# to resist to failures as otherwise an orphaned master can't be failed over
+# in case of failure if it has no working slaves.
+#
+# Slaves migrate to orphaned masters only if there are still at least a
+# given number of other working slaves for their old master. This number
+# is the "migration barrier". A migration barrier of 1 means that a slave
+# will migrate only if there is at least 1 other working slave for its master
+# and so forth. It usually reflects the number of slaves you want for every
+# master in your cluster.
+#
+# Default is 1 (slaves migrate only if their masters remain with at least
+# one slave). To disable migration just set it to a very large value.
+# A value of 0 can be set but is useful only for debugging and dangerous
+# in production.
+#
+# cluster-migration-barrier 1
+
+# By default Redis Cluster nodes stop accepting queries if they detect there
+# is at least an hash slot uncovered (no available node is serving it).
+# This way if the cluster is partially down (for example a range of hash slots
+# are no longer covered) all the cluster becomes, eventually, unavailable.
+# It automatically returns available as soon as all the slots are covered again.
+#
+# However sometimes you want the subset of the cluster which is working,
+# to continue to accept queries for the part of the key space that is still
+# covered. In order to do so, just set the cluster-require-full-coverage
+# option to no.
+#
+# cluster-require-full-coverage yes
+
+# In order to setup your cluster make sure to read the documentation
+# available at http://redis.io web site.
+
+################################## SLOW LOG ###################################
+
+# The Redis Slow Log is a system to log queries that exceeded a specified
+# execution time. The execution time does not include the I/O operations
+# like talking with the client, sending the reply and so forth,
+# but just the time needed to actually execute the command (this is the only
+# stage of command execution where the thread is blocked and can not serve
+# other requests in the meantime).
+#
+# You can configure the slow log with two parameters: one tells Redis
+# what is the execution time, in microseconds, to exceed in order for the
+# command to get logged, and the other parameter is the length of the
+# slow log. When a new command is logged the oldest one is removed from the
+# queue of logged commands.
+
+# The following time is expressed in microseconds, so 1000000 is equivalent
+# to one second. Note that a negative number disables the slow log, while
+# a value of zero forces the logging of every command.
+slowlog-log-slower-than 10000
+
+# There is no limit to this length. Just be aware that it will consume memory.
+# You can reclaim memory used by the slow log with SLOWLOG RESET.
+slowlog-max-len 128
+
+################################ LATENCY MONITOR ##############################
+
+# The Redis latency monitoring subsystem samples different operations
+# at runtime in order to collect data related to possible sources of
+# latency of a Redis instance.
+#
+# Via the LATENCY command this information is available to the user that can
+# print graphs and obtain reports.
+#
+# The system only logs operations that were performed in a time equal or
+# greater than the amount of milliseconds specified via the
+# latency-monitor-threshold configuration directive. When its value is set
+# to zero, the latency monitor is turned off.
+#
+# By default latency monitoring is disabled since it is mostly not needed
+# if you don't have latency issues, and collecting data has a performance
+# impact, that while very small, can be measured under big load. Latency
+# monitoring can easily be enabled at runtime using the command
+# "CONFIG SET latency-monitor-threshold <milliseconds>" if needed.
+latency-monitor-threshold 0
+
+############################# EVENT NOTIFICATION ##############################
+
+# Redis can notify Pub/Sub clients about events happening in the key space.
+# This feature is documented at http://redis.io/topics/notifications
+#
+# For instance if keyspace events notification is enabled, and a client
+# performs a DEL operation on key "foo" stored in the Database 0, two
+# messages will be published via Pub/Sub:
+#
+# PUBLISH __keyspace@0__:foo del
+# PUBLISH __keyevent@0__:del foo
+#
+# It is possible to select the events that Redis will notify among a set
+# of classes. Every class is identified by a single character:
+#
+#  K     Keyspace events, published with __keyspace@<db>__ prefix.
+#  E     Keyevent events, published with __keyevent@<db>__ prefix.
+#  g     Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ...
+#  $     String commands
+#  l     List commands
+#  s     Set commands
+#  h     Hash commands
+#  z     Sorted set commands
+#  x     Expired events (events generated every time a key expires)
+#  e     Evicted events (events generated when a key is evicted for maxmemory)
+#  A     Alias for g$lshzxe, so that the "AKE" string means all the events.
+#
+#  The "notify-keyspace-events" takes as argument a string that is composed
+#  of zero or multiple characters. The empty string means that notifications
+#  are disabled.
+#
+#  Example: to enable list and generic events, from the point of view of the
+#           event name, use:
+#
+#  notify-keyspace-events Elg
+#
+#  Example 2: to get the stream of the expired keys subscribing to channel
+#             name __keyevent@0__:expired use:
+#
+#  notify-keyspace-events Ex
+#
+#  By default all notifications are disabled because most users don't need
+#  this feature and the feature has some overhead. Note that if you don't
+#  specify at least one of K or E, no events will be delivered.
+notify-keyspace-events ""
+
+############################### ADVANCED CONFIG ###############################
+
+# Hashes are encoded using a memory efficient data structure when they have a
+# small number of entries, and the biggest entry does not exceed a given
+# threshold. These thresholds can be configured using the following directives.
+hash-max-ziplist-entries 512
+hash-max-ziplist-value 64
+
+# Lists are also encoded in a special way to save a lot of space.
+# The number of entries allowed per internal list node can be specified
+# as a fixed maximum size or a maximum number of elements.
+# For a fixed maximum size, use -5 through -1, meaning:
+# -5: max size: 64 Kb  <-- not recommended for normal workloads
+# -4: max size: 32 Kb  <-- not recommended
+# -3: max size: 16 Kb  <-- probably not recommended
+# -2: max size: 8 Kb   <-- good
+# -1: max size: 4 Kb   <-- good
+# Positive numbers mean store up to _exactly_ that number of elements
+# per list node.
+# The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size),
+# but if your use case is unique, adjust the settings as necessary.
+list-max-ziplist-size -2
+
+# Lists may also be compressed.
+# Compress depth is the number of quicklist ziplist nodes from *each* side of
+# the list to *exclude* from compression.  The head and tail of the list
+# are always uncompressed for fast push/pop operations.  Settings are:
+# 0: disable all list compression
+# 1: depth 1 means "don't start compressing until after 1 node into the list,
+#    going from either the head or tail"
+#    So: [head]->node->node->...->node->[tail]
+#    [head], [tail] will always be uncompressed; inner nodes will compress.
+# 2: [head]->[next]->node->node->...->node->[prev]->[tail]
+#    2 here means: don't compress head or head->next or tail->prev or tail,
+#    but compress all nodes between them.
+# 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail]
+# etc.
+list-compress-depth 0
+
+# Sets have a special encoding in just one case: when a set is composed
+# of just strings that happen to be integers in radix 10 in the range
+# of 64 bit signed integers.
+# The following configuration setting sets the limit in the size of the
+# set in order to use this special memory saving encoding.
+set-max-intset-entries 512
+
+# Similarly to hashes and lists, sorted sets are also specially encoded in
+# order to save a lot of space. This encoding is only used when the length and
+# elements of a sorted set are below the following limits:
+zset-max-ziplist-entries 128
+zset-max-ziplist-value 64
+
+# HyperLogLog sparse representation bytes limit. The limit includes the
+# 16 bytes header. When an HyperLogLog using the sparse representation crosses
+# this limit, it is converted into the dense representation.
+#
+# A value greater than 16000 is totally useless, since at that point the
+# dense representation is more memory efficient.
+#
+# The suggested value is ~ 3000 in order to have the benefits of
+# the space efficient encoding without slowing down too much PFADD,
+# which is O(N) with the sparse encoding. The value can be raised to
+# ~ 10000 when CPU is not a concern, but space is, and the data set is
+# composed of many HyperLogLogs with cardinality in the 0 - 15000 range.
+hll-sparse-max-bytes 3000
+
+# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
+# order to help rehashing the main Redis hash table (the one mapping top-level
+# keys to values). The hash table implementation Redis uses (see dict.c)
+# performs a lazy rehashing: the more operation you run into a hash table
+# that is rehashing, the more rehashing "steps" are performed, so if the
+# server is idle the rehashing is never complete and some more memory is used
+# by the hash table.
+#
+# The default is to use this millisecond 10 times every second in order to
+# actively rehash the main dictionaries, freeing memory when possible.
+#
+# If unsure:
+# use "activerehashing no" if you have hard latency requirements and it is
+# not a good thing in your environment that Redis can reply from time to time
+# to queries with 2 milliseconds delay.
+#
+# use "activerehashing yes" if you don't have such hard requirements but
+# want to free memory asap when possible.
+activerehashing yes
+
+# The client output buffer limits can be used to force disconnection of clients
+# that are not reading data from the server fast enough for some reason (a
+# common reason is that a Pub/Sub client can't consume messages as fast as the
+# publisher can produce them).
+#
+# The limit can be set differently for the three different classes of clients:
+#
+# normal -> normal clients including MONITOR clients
+# slave  -> slave clients
+# pubsub -> clients subscribed to at least one pubsub channel or pattern
+#
+# The syntax of every client-output-buffer-limit directive is the following:
+#
+# client-output-buffer-limit <class> <hard limit> <soft limit> <soft seconds>
+#
+# A client is immediately disconnected once the hard limit is reached, or if
+# the soft limit is reached and remains reached for the specified number of
+# seconds (continuously).
+# So for instance if the hard limit is 32 megabytes and the soft limit is
+# 16 megabytes / 10 seconds, the client will get disconnected immediately
+# if the size of the output buffers reach 32 megabytes, but will also get
+# disconnected if the client reaches 16 megabytes and continuously overcomes
+# the limit for 10 seconds.
+#
+# By default normal clients are not limited because they don't receive data
+# without asking (in a push way), but just after a request, so only
+# asynchronous clients may create a scenario where data is requested faster
+# than it can read.
+#
+# Instead there is a default limit for pubsub and slave clients, since
+# subscribers and slaves receive data in a push fashion.
+#
+# Both the hard or the soft limit can be disabled by setting them to zero.
+client-output-buffer-limit normal 0 0 0
+client-output-buffer-limit slave 256mb 64mb 60
+client-output-buffer-limit pubsub 32mb 8mb 60
+
+# Redis calls an internal function to perform many background tasks, like
+# closing connections of clients in timeout, purging expired keys that are
+# never requested, and so forth.
+#
+# Not all tasks are performed with the same frequency, but Redis checks for
+# tasks to perform according to the specified "hz" value.
+#
+# By default "hz" is set to 10. Raising the value will use more CPU when
+# Redis is idle, but at the same time will make Redis more responsive when
+# there are many keys expiring at the same time, and timeouts may be
+# handled with more precision.
+#
+# The range is between 1 and 500, however a value over 100 is usually not
+# a good idea. Most users should use the default of 10 and raise this up to
+# 100 only in environments where very low latency is required.
+hz 10
+
+# When a child rewrites the AOF file, if the following option is enabled
+# the file will be fsync-ed every 32 MB of data generated. This is useful
+# in order to commit the file to the disk more incrementally and avoid
+# big latency spikes.
+aof-rewrite-incremental-fsync yes
diff --git a/ansible/roles/redis-multiprocess/templates/systemd.j2 b/ansible/roles/redis-multiprocess/templates/systemd.j2
new file mode 100644
index 0000000000..b9633f9183
--- /dev/null
+++ b/ansible/roles/redis-multiprocess/templates/systemd.j2
@@ -0,0 +1,14 @@
+[Unit]
+Description=Redis In-Memory Data Store
+After=network.target
+
+[Service]
+User={{ redis_user }}
+Group={{ redis_user }}
+ExecStart={{ redis_dir }}/redis-stable/src/redis-server {{ redis_dir }}/redis-stable/redis-_port_.conf
+ExecStop={{ redis_dir }}/redis-stable/src/redis-cli -p _port_ shutdown
+Restart=always
+Type=notify
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/redis-restore/defaults/main.yml b/ansible/roles/redis-restore/defaults/main.yml
index 8d7b35064f..dfa157a310 100644
--- a/ansible/roles/redis-restore/defaults/main.yml
+++ b/ansible/roles/redis-restore/defaults/main.yml
@@ -1,2 +1,4 @@
-redis_backup_azure_container_name: dp-redis-backup
 analytics_user_home: /home/analytics
+
+cloud_storage_dpredisbackup_bucketname: "{{ cloud_storage_management_bucketname }}"
+cloud_storage_dpredisbackup_foldername: dp-redis-backup
diff --git a/ansible/roles/redis-restore/meta/main.yml b/ansible/roles/redis-restore/meta/main.yml
deleted file mode 100644
index a124d4f7cb..0000000000
--- a/ansible/roles/redis-restore/meta/main.yml
+++ /dev/null
@@ -1,2 +0,0 @@
-dependencies:
-  - azure-cli
diff --git a/ansible/roles/redis-restore/tasks/main.yml b/ansible/roles/redis-restore/tasks/main.yml
index b2cf44f3b1..a695f11e62 100644
--- a/ansible/roles/redis-restore/tasks/main.yml
+++ b/ansible/roles/redis-restore/tasks/main.yml
@@ -1,9 +1,42 @@
 ---
 
-- name: Download backup file
-  shell: "az storage blob download --container-name {{ redis_backup_azure_container_name }} --file {{ redis_restore_file_name }} --name {{ redis_restore_file_name }} --account-name {{sunbird_management_storage_account_name}} --account-key {{sunbird_management_storage_account_key}}"
-  args:
-    chdir: /tmp/ 
+- name: download a file from azure storage
+  include_role:
+    name: azure-cloud-storage
+    tasks_from: blob-download.yml
+  vars:
+    blob_container_name: "{{ cloud_storage_dpredisbackup_foldername }}"
+    blob_file_name: "{{ redis_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ redis_restore_file_name }}"
+    storage_account_name: "{{ cloud_management_storage_accountname }}"
+    storage_account_key: "{{ cloud_management_storage_secret }}"
+  when: cloud_service_provider == "azure"
+
+- name: download file from aws s3
+  include_role:
+    name: aws-cloud-storage
+    tasks_from: download.yml
+  vars:
+    s3_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    aws_access_key_id: "{{ cloud_management_storage_accountname }}"
+    aws_secret_access_key: "{{ cloud_management_storage_secret }}"
+    aws_default_region: "{{ cloud_public_storage_region }}"
+    local_file_or_folder_path: "/tmp/{{ redis_restore_file_name }}"
+    s3_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_restore_file_name }}"
+  when: cloud_service_provider == "aws"
+
+- name: download file from gcloud storage
+  include_role:
+    name: gcp-cloud-storage
+    tasks_from: download.yml
+  vars:
+    gcp_storage_service_account_name: "{{ cloud_management_storage_accountname }}"
+    gcp_storage_key_file: "{{ cloud_management_storage_secret }}"
+    gcp_bucket_name: "{{ cloud_storage_dpredisbackup_bucketname }}"
+    gcp_path: "{{ cloud_storage_dpredisbackup_foldername }}/{{ redis_restore_file_name }}"
+    dest_file_name: "{{ redis_restore_file_name }}"
+    local_file_or_folder_path: "/tmp/{{ redis_restore_file_name }}"
+  when: cloud_service_provider == "gcloud"
 
 - name: stop redis to take backup 
   become: yes
diff --git a/ansible/roles/redis/defaults/main.yml b/ansible/roles/redis/defaults/main.yml
index 9d07c5517c..38ea45f8da 100644
--- a/ansible/roles/redis/defaults/main.yml
+++ b/ansible/roles/redis/defaults/main.yml
@@ -1,11 +1,10 @@
 redis_user: analytics
-analytics_user_home: /home/{{ redis_user }}
-redis_home: "{{ analytics_user_home }}/redis-stable"
-redis_tar_ball: http://download.redis.io/releases/redis-stable.tar.gz
+analytics_redis_user_home: /home/{{ redis_user }}
+redis_home: "{{ analytics_redis_user_home }}/redis-{{ redis_version }}"
+redis_tar_ball: "http://download.redis.io/releases/redis-{{ redis_version }}.tar.gz"
 dp_redis_max_memory: 2gb
-redis_dir: "{{ analytics_user_home }}"
+redis_dir: "{{ analytics_redis_user_home }}"
 redis_dump_rdb_time_in_seconds: 900
 redis_dump_rdb_keys_to_save_state: 100
-
-
-
+redis_data_dir: /data
+redis_version: 6.2.5
diff --git a/ansible/roles/redis/tasks/main.yml b/ansible/roles/redis/tasks/main.yml
index ed2821dc1a..11b9409ae0 100755
--- a/ansible/roles/redis/tasks/main.yml
+++ b/ansible/roles/redis/tasks/main.yml
@@ -1,14 +1,30 @@
 ---
 # tasks file for redis
+- name: change the data dir ownership
+  file:
+    path: /data
+    state: directory
+    owner: "{{ redis_user }}"
+    group: "{{ redis_user }}"
+    mode: 0755
+  become: yes
+
+- name: create the redis dir
+  become: yes
+  become_user: "{{ redis_user }}"
+  file:
+    path: "{{ analytics_redis_user_home }}/redis"
+    state: directory
+
 - name: download redis zipfile
   become: yes
   become_user: "{{ redis_user }}"
-  get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-stable.tar.gz timeout=50 force=no owner={{ redis_user }}
+  get_url: url={{ redis_tar_ball }} dest={{ redis_dir }}/redis-{{ redis_version }}.tar.gz timeout=50 force=no owner={{ redis_user }}
 
 - name: unzip
   become: yes
   become_user: "{{ redis_user }}"
-  unarchive: src={{ redis_dir }}/redis-stable.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }}
+  unarchive: src={{ redis_dir }}/redis-{{ redis_version }}.tar.gz dest={{ redis_dir }} copy=no group={{ redis_user }} owner={{ redis_user }} creates={{ redis_home }}
 
 - name: Change ownership of redis installation
   become: yes
@@ -16,7 +32,7 @@
 
 - name: redis conf
   become: yes
-  template: src=redis.conf.j2 dest={{ redis_dir }}/redis-stable/redis.conf
+  template: src=redis.conf.j2 dest={{ redis_dir }}/redis-{{ redis_version }}/redis.conf
 
 - name: Install required utils
   apt: name={{ item }} state=present
@@ -25,7 +41,6 @@
     - make
     - gcc
     - build-essential
-    - tcl8.5
 
 - name: Go to the folder and run make
   become: yes
diff --git a/ansible/roles/redis/templates/redis.conf.j2 b/ansible/roles/redis/templates/redis.conf.j2
index f1d0bbdd7f..e7c725c58d 100644
--- a/ansible/roles/redis/templates/redis.conf.j2
+++ b/ansible/roles/redis/templates/redis.conf.j2
@@ -160,7 +160,7 @@ loglevel notice
 # Specify the log file name. Also the empty string can be used to force
 # Redis to log on the standard output. Note that if you use standard
 # output for logging but daemonize, logs will be sent to /dev/null
-logfile "{{ redis_dir }}/redis-stable/redis.log"
+logfile "{{ redis_data_dir }}/redis.log"
 
 
 # To enable logging to the system logger, just set 'syslog-enabled' to yes,
@@ -246,7 +246,7 @@ dbfilename dump.rdb
 # The Append Only File will also be created inside this directory.
 #
 # Note that you must specify a directory here, not a file name.
-dir "{{ redis_dir }}/redis-stable/"
+dir "{{redis_data_dir}}"
 
 
 ################################# REPLICATION #################################
diff --git a/ansible/roles/redis/templates/systemd.j2 b/ansible/roles/redis/templates/systemd.j2
index 87f7a2dab4..9f9cf59fde 100644
--- a/ansible/roles/redis/templates/systemd.j2
+++ b/ansible/roles/redis/templates/systemd.j2
@@ -5,10 +5,10 @@ After=network.target
 [Service]
 User={{ redis_user }}
 Group={{ redis_user }}
-ExecStart={{ redis_dir }}/redis-stable/src/redis-server {{ redis_dir }}/redis-stable/redis.conf
-ExecStop={{ redis_dir }}/redis-stable/src/redis-cli shutdown
+ExecStart={{ redis_dir }}/redis-{{ redis_version }}/src/redis-server {{ redis_dir }}/redis-{{ redis_version }}/redis.conf
+ExecStop={{ redis_dir }}/redis-{{ redis_version }}/src/redis-cli shutdown
 Restart=always
-Type=notify
+Type=forking
 
 [Install]
 WantedBy=multi-user.target
diff --git a/ansible/roles/samza-jobs/defaults/main.yml b/ansible/roles/samza-jobs/defaults/main.yml
deleted file mode 100644
index 8f2ac30540..0000000000
--- a/ansible/roles/samza-jobs/defaults/main.yml
+++ /dev/null
@@ -1,86 +0,0 @@
----
-samza_jobs_dir: /home/hduser/samza-jobs/{{ env }}
-job_status_file: /home/hduser/samza-jobs/{{ env }}/extract/job_status
-yarn_path: /usr/local/hadoop/bin
-lpdeploy: no
-dpdeploy: no
-hadoop_version: 2.7.2
-# Defaults
-telemetry_extractor_yarn_container_count: 1
-telemetry_validator_yarn_container_count: 1
-telemetry_de_duplication_yarn_container_count: 1
-telemetry_router_yarn_container_count: 1
-telemetry_reverse_search_yarn_container_count: 1
-telemetry_object_de_normalization_yarn_container_count: 1
-telemetry_es_indexer_yarn_container_count: 1
-telemetry_location_updater_yarn_container_count: 1
-events_router_yarn_container_count: 1
-denormalization_yarn_container_count: 1
-druid_events_validator_yarn_container_count: 1
-redis_updater_yarn_container_count: 1
-druidprocessor_yarn_container_count: 1
-device_profile_updater_yarn_container_count: 1
-assessment_aggregator_yarn_container_count: 1
-telemetry_derived_de_duplication_yarn_container_count: 1
-events_flattener_yarn_container_count: 1
-telemetry_redacter_yarn_container_count: 1
-
-
-# samza jobs checkpoint replication factor
-samza_checkpoint_replication_factor: 1
-
-max_iteration_count_for_samza_job: 2
-location_db_redis_key_expiry_seconds: 86400
-cache_unresolved_location_key_expiry_seconds: 3600
-redis_host: "{{ groups['redis'][0] }}"
-redis_port: 6379
-metadata_redis_host: "{{ groups['redis'][0] }}"
-metadata_redis_port: 6379
-__yarn_port__: 8000
-delayInMilliSeconds: 60000
-retryTimeInMilliSeconds: 10000
-retry_backoff_base_in_seconds: 10
-bypass_reverse_search: true
-retry_limit: 4
-retry_limit_enable: true
-environment_id: 10000000
-tr_secondary_route_events: "LOG,ERROR"
-google_vision_tagging: false
-google_api_key: "123"
-cassandra_port: 9042
-postgres_password: "{{ postgres.db_password }}"
-postgres_db: "{{ postgres.db_name }}"
-postgres_user: "{{ postgres.db_username }}"
-postgres_host: "{{ postgres.db_url }}"
-postgres_port: "{{ postgres.db_port }}"
-es_port: 9200
-
-middleware_cassandra_keyspace: sunbird
-middleware_cassandra_user_table: user
-middleware_cassandra_location_table: location
-middleware_cassandra_courses_keyspace: sunbird_courses
-middleware_cassandra_assessment_aggregator_table: assessment_aggregator
-middleware_cassandra_assessment_question_type : question
-
-telemetry_extractor_consumer_fetch_max_bytes: 5242880
-telemetry_extractor_container_memory_mb: 1024
-telemetry_extractor_messages_fetch_threshold: 3000
-consumer_fetch_max_bytes: 1572864
-producer_max_request_size_bytes: 1572864
-assessment_aggregator_messages_fetch_threshold: 3000
-assessment_aggregator_consumer_fetch_max_bytes: 5242880
-
-dedup_include_env_producer_ids: ""
-
-content_metadata_fields: "name, objectType, contentType, mediaType, language, medium, subject, gradeLevel, mimeType, createdBy, createdFor, framework, board, status, pkgVersion, lastSubmittedOn, lastUpdatedOn, lastPublishedOn, channel"
-user_metadata_fields: "usertype, grade, subject, language, state, district, usersignintype,userlogintype"
-dialcode_metadata_fields: "identifier, channel, batchcode, publisher, generated_on, published_on, status"
-
-user_signin_type_default : Anonymous
-user_login_type_default : NA
-
-#Purpose: Content cache update job uses the dialcode api
-dialcode_api_host: "{{ dialcode_host }}"
-dialcode_api_endpoint: "{{ dialcode_endpoint }}"
-dialcode_api_auth_key: "{{ dialcode_api_key }}"
-dedup_exclude_eids: "LOG,ERROR"
diff --git a/ansible/roles/samza-jobs/tasks/main.yml b/ansible/roles/samza-jobs/tasks/main.yml
deleted file mode 100644
index 3e28039576..0000000000
--- a/ansible/roles/samza-jobs/tasks/main.yml
+++ /dev/null
@@ -1,73 +0,0 @@
- - name: Create Directory for Jobs
-   file: path={{ item }} owner=hduser group=hadoop recurse=yes state=directory
-   with_items:
-     - "{{ samza_jobs_dir }}"
-     - "{{ samza_jobs_dir }}/extract"
-
- - name: Get the application id to kill the app
-   shell: "{{ yarn_path }}/yarn application --list | grep -i {{ item }} | awk '{print $1}'"
-   with_items: "{{ job_names_to_kill.split(',')|list }}"
-   register: appid
-
- - name: Kill the mentioned applications
-   shell: "{{ yarn_path }}/yarn application -kill {{ item.stdout }}"
-   with_items:
-     - "{{ appid['results'] }}"
-   when: item.stdout | length > 0
-
- - name: find the existing file names to remove
-   find:
-     paths: "{{ samza_jobs_dir }}"
-     patterns: "{{ job_names['%s'|format(item)].job_file_name }}*"
-     recurse: yes
-   with_items: "{{ job_names_to_kill.split(',') }}"
-   register: existing_files
-
- - name: remove the files under "{{ samza_jobs_dir }}" directory
-   command: rm -rf "{{ item.path | basename }}"
-   with_items: "{{ existing_files | json_query('results[].files[]') }}"
-   args:
-      chdir: "{{ samza_jobs_dir }}"
-
- - name: remove the files under "{{ samza_jobs_dir }}/extract" directory
-   command: rm -rf "{{ item.path | basename }}"
-   with_items: "{{ existing_files | json_query('results[].files[]') }}"
-   args:
-      chdir: "{{ samza_jobs_dir }}/extract"
-
- - name: copy new jobs tar ball
-   copy: src={{ item }} dest={{ samza_jobs_dir }}/ force=no owner=hduser group=hadoop
-   with_fileglob:
-     - ../defaults/jobs/*
-   register: new_jobs
-
- - name: Create directory to extract new jobs
-   file: path="{{ samza_jobs_dir }}/extract/{{ item }}" owner=hduser group=hadoop recurse=yes state=directory
-   with_items:
-     - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}"
-
- - name: extract new jobs
-   unarchive:
-         src: "{{ samza_jobs_dir }}/{{ item }}"
-         dest: "{{ samza_jobs_dir }}/extract/{{ item }}"
-         remote_src: yes
-   with_items:
-     - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}"
-
- - name: Get all new jobs config
-   shell: "ls -d -1 {{ samza_jobs_dir }}/extract/{{ item }}/config/*.properties"
-   register: config_files
-   with_items:
-     - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}"
-
- - name: update environment specific details in new job configs
-   replace: dest="{{ item[1].stdout }}" regexp="{{ item[0].key }}" replace="{{ item[0].value }}"
-   with_nested:
-     - [{key: "__yarn_host__", value: "{{ __yarn_host__ }}"}, {key: "__yarn_port__", value: "{{ __yarn_port__ }}"}, {key: "__env__", value: "{{ env }}" }, {key: "__env_name__", value: "{{ env_name }}" }, {key: "__zookeepers__", value: "{{ zookeepers }}"}, {key: "__kafka_brokers__", value: "{{ kafka_brokers }}"}, {key: "__delayInMilliSeconds__", value: "{{ delayInMilliSeconds }}" }, {key: "__retryTimeInMilliSeconds__", value: "{{ retryTimeInMilliSeconds }}" }, {key: "__bypass_reverse_search__", value: "{{ bypass_reverse_search }}" }, {key: "__retryBackoffBaseInSeconds__", value: "{{ retry_backoff_base_in_seconds }}" }, {key: "__retryLimit__", value: "{{ retry_limit }}" }, {key: "__retryLimitEnable__", value: "{{ retry_limit_enable }}" },  {key: "__google_api_key__", value: "{{ google_api_key }}" }, {key: "__searchServiceEndpoint__", value: "{{ search_service_endpoint }}" }, {key: "__environment_id__", value: "{{ environment_id }}"}, {key: "__google_vision_tagging__", value: "{{ google_vision_tagging }}"}, {key: "__cassandra_host__", value: "{{ cassandra_host }}"},{key: "__cassandra_port__", value: "{{ cassandra_port }}"}, {key: "__content_to_vec_url__", value: "{{ content_to_vec_url }}"},{key: "__max_iteration_count_for_samza_job__", value: "{{ max_iteration_count_for_samza_job }}"},{key: "__device_profile_updater_yarn_container_count__", value: "{{ device_profile_updater_yarn_container_count }}"},{key: "__telemetry_extractor_yarn_container_count__", value: "{{ telemetry_extractor_yarn_container_count }}"},{key: "__telemetry_validator_yarn_container_count__", value: "{{ telemetry_validator_yarn_container_count }}"},{key: "__telemetry_de_duplication_yarn_container_count__", value: "{{ telemetry_de_duplication_yarn_container_count }}"},{key: "__telemetry_router_yarn_container_count__", value: "{{ telemetry_router_yarn_container_count }}"},{key: "__telemetry_reverse_search_yarn_container_count__", value: "{{ telemetry_reverse_search_yarn_container_count }}"},{key: "__telemetry_object_de_normalization_yarn_container_count__", value: "{{ telemetry_object_de_normalization_yarn_container_count }}"}, {key: "__events_router_yarn_container_count__", value: "{{ events_router_yarn_container_count }}"}, {key: "__ingestion_zookeepers__", value: "{{ ingestion_zookeepers }}"}, {key: "__ingestion_kafka_brokers__", value: "{{ ingestion_kafka_brokers }}"}, {key: "__tr_secondary_route_events__", value: "{{ tr_secondary_route_events }}"},{key: "__telemetry_schema_path__", value: "{{ telemetry_schema_path }}"},{key: "__default_channel__", value: "{{ default_channel }}"},{key: "__telemetry_location_updater_yarn_container_count__", value: "{{ telemetry_location_updater_yarn_container_count }}"},{key: "__channelSearchServiceEndpoint__", value: "{{ channelSearchServiceEndpoint }}"},{key: "__locationSearchServiceEndpoint__", value: "{{ locationSearchServiceEndpoint }}"},{key: "__searchServiceAuthorizationToken__", value: "{{ searchServiceAuthorizationToken }}"},{key: "__redis_host__", value: "{{ redis_host }}"}, {key: "__redis_port__", value: "{{ redis_port }}"},{key: "__location_db_redis_key_expiry_seconds__", value: "{{ location_db_redis_key_expiry_seconds }}"}, {key: "__cache_unresolved_location_key_expiry_seconds__", value: "{{ cache_unresolved_location_key_expiry_seconds }}"}, {key: "__middleware_cassandra_host__", value: "{{ core_cassandra_host }}"},{key: "__middleware_cassandra_port__", value: "{{ cassandra_port }}"},{key: "__middleware_cassandra_keyspace__", value: "{{ middleware_cassandra_keyspace }}"}, {key: "__middleware_cassandra_user_table__", value: "{{ middleware_cassandra_user_table }}"}, {key: "__middleware_cassandra_location_table__", value: "{{ middleware_cassandra_location_table }}"}, {key: "__telemetry_extractor_consumer_fetch_max_bytes__", value: "{{ telemetry_extractor_consumer_fetch_max_bytes }}"}, {key: "__telemetry_extractor_container_memory_mb__", value: "{{ telemetry_extractor_container_memory_mb }}"}, {key: "__telemetry_extractor_messages_fetch_threshold__", value: "{{ telemetry_extractor_messages_fetch_threshold }}"}, {key: "__denormalization_yarn_container_count__", value: "{{ denormalization_yarn_container_count }}"}, {key: "__druidprocessor_yarn_container_count__", value: "{{ druidprocessor_yarn_container_count }}"}, {key: "__content_metadata_fields__", value: "{{ content_metadata_fields }}"}, {key: "__user_metadata_fields__", value: "{{ user_metadata_fields }}"}, {key: "__dialcode_metadata_fields__", value: "{{ dialcode_metadata_fields }}"}, {key: "__druid_events_validator_yarn_container_count__", value: "{{ druid_events_validator_yarn_container_count }}"}, {key: "__redis_updater_yarn_container_count__", value: "{{ redis_updater_yarn_container_count }}"}, {key: "__samza_checkpoint_replication_factor__", value: "{{ samza_checkpoint_replication_factor }}"}, {key: "__consumer_fetch_max_bytes__", value: "{{ consumer_fetch_max_bytes }}"}, {key: "__producer_max_request_size_bytes__", value: "{{ producer_max_request_size_bytes }}"}, {key: "__dedup_include_producer_ids__", value: "{{ dedup_include_env_producer_ids }}"}, {key: "__user_sigin_type_default__", value: "{{ user_signin_type_default }}"}, {key: "__user_login_type_default__", value: "{{ user_login_type_default }}"}, {key: "__middleware_cassandra__courses_keyspace__", value: "{{ middleware_cassandra_courses_keyspace }}"}, {key: "__middleware_cassandra_assessment_aggregator_table__", value: "{{ middleware_cassandra_assessment_aggregator_table }}"}, {key: "__middleware_cassandra_assessment_question_type__", value: "{{ middleware_cassandra_assessment_question_type }}"}, {key: "__assessment_aggregator_yarn_container_count__", value: "{{ assessment_aggregator_yarn_container_count }}"}, {key: "__assessment_aggregator_consumer_fetch_max_bytes__", value: "{{ assessment_aggregator_consumer_fetch_max_bytes }}"}, {key: "__postgres_db__", value: "{{ postgres_db }}"}, {key: "__postgres_user__", value: "{{ postgres_user }}"}, {key: "__postgres_password__", value: "{{ postgres_password }}"}, {key: "__postgres_host__", value: "{{ postgres_host }}"}, {key: "__postgres_port__", value: "{{ postgres_port }}"}, {key: "__assessment_aggregator_messages_fetch_threshold__", value: "{{ assessment_aggregator_messages_fetch_threshold }}"},{key: "__telemetry_derived_de_duplication_yarn_container_count__", value: "{{ telemetry_derived_de_duplication_yarn_container_count }}"},{key: "__events_flattener_yarn_container_count__", value: "{{ events_flattener_yarn_container_count}}"},{key: "__metadata_redis_host__", value: "{{ metadata_redis_host }}"}, {key: "__metadata_redis_port__", value: "{{ metadata_redis_port }}"},{key: "__dedup_exclude_eids__", value: "{{ dedup_exclude_eids }}"},{key: "__telemetry_redacter_yarn_container_count__", value: "{{ telemetry_redacter_yarn_container_count }}"},{key: "__dialcode_host__", value: "{{ dialcode_api_host }}"}, {key: "__dialcode_api_endpoint__", value: "{{ dialcode_api_endpoint }}"}, {key: "__dialcode_authorization_key__", value: "{{ dialcode_api_auth_key }}"}]
-     - "{{ config_files | json_query('results[]') }}"
-
- - name: Start the jobs
-   shell: "{{ samza_jobs_dir }}/extract/{{ item.0 }}/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path={{ item.1.stdout }}"
-   with_together:
-     - "{{ new_jobs | json_query('results[].invocation.module_args.original_basename') }}"
-     - "{{ config_files | json_query('results[]') }}"
diff --git a/ansible/roles/secor-deploy/tasks/main.yml b/ansible/roles/secor-deploy/tasks/main.yml
index 8411b28664..cefb2e3ba7 100644
--- a/ansible/roles/secor-deploy/tasks/main.yml
+++ b/ansible/roles/secor-deploy/tasks/main.yml
@@ -25,4 +25,4 @@
     owner: "{{ analytics_user }}"
     group: "{{ analytics_user }}"
   with_fileglob:
-    - "artifacts/target/secor-*-SNAPSHOT-bin.tar.gz"
+    - "artifacts/target/secor-{{ secor.artifact_ver }}-bin.tar.gz"
diff --git a/ansible/roles/secor-deploy/templates/secor-service.j2 b/ansible/roles/secor-deploy/templates/secor-service.j2
index 2fd50cb136..cd5578312d 100644
--- a/ansible/roles/secor-deploy/templates/secor-service.j2
+++ b/ansible/roles/secor-deploy/templates/secor-service.j2
@@ -29,7 +29,7 @@ function start() {
   fi
 
   cd /mount/$SECOR_NAME
-    nohup /usr/bin/java -Xms256M -Xmx1000M -ea -Duser.timezone=UTC -Dsecor_group=$SECOR_NAME -Dlog4j.configuration=log4j.properties -Dconfig=secor.partition.properties -cp secor-0.25-SNAPSHOT.jar:lib/* com.pinterest.secor.main.ConsumerMain > /mount/secor/logs/$SECOR_NAME-service.log 2>&1 &
+    nohup /usr/bin/java -Xms256M -Xmx1000M -ea -Duser.timezone=UTC -Dsecor_group=$SECOR_NAME -Dlog4j.configuration=log4j.properties -Dconfig=secor.partition.properties -cp secor-{{ secor.artifact_ver }}.jar:lib/* com.pinterest.secor.main.ConsumerMain > /mount/secor/logs/$SECOR_NAME-service.log 2>&1 &
   pid=$!
   sleep 1
 
diff --git a/ansible/roles/secor-fetch-logs/tasks/main.yml b/ansible/roles/secor-fetch-logs/tasks/main.yml
index 1fba0c383d..1b35630fc5 100644
--- a/ansible/roles/secor-fetch-logs/tasks/main.yml
+++ b/ansible/roles/secor-fetch-logs/tasks/main.yml
@@ -1,7 +1,6 @@
 - name: Fetch secor logs
-  sudo: no
   synchronize: src="{{ item }}" dest="./secor/" mode=pull recursive=yes rsync_path=rsync
   with_items:
     - "/mount/secor/logs"
   tags:
-    - fetch-logs
\ No newline at end of file
+    - fetch-logs
diff --git a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
index 2b23e0a15e..a9edd50a3c 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
+++ b/ansible/roles/secor-telemetry-backup-deploy/defaults/main.yml
@@ -18,11 +18,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.failed"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9997"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -30,6 +29,7 @@ secor_service_name:
     service_description: "Failed Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   raw-telemetry-backup:
     consumer_group: "{{ env }}.telemetry.raw.backup"
@@ -37,11 +37,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.raw"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9994"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -49,6 +48,7 @@ secor_service_name:
     service_description: "Raw Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   unique-telemetry-backup:
     consumer_group: "{{ env }}.telemetry.unique.backup"
@@ -56,11 +56,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.unique"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9993"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -68,6 +67,7 @@ secor_service_name:
     service_description: "Unique Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   denorm-events-backup:
     consumer_group: "{{ env }}.telemetry.denorm.backup"
@@ -75,11 +75,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.denorm"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9990"
     partition_prefix_enabled: "true"
     partition_prefix_key: "eid"
     partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"summary\",\"ME_WORKFLOW_SUMMARY\":\"summary\",\"ME_DEIVCE_SUMMARY\":\"summary\",\"ME_ITEM_SUMMARY\":\"summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"summary\",\"ME_ITEM_USAGE_SUMMARY\":\"summary\",\"ME_USAGE_SUMMARY\":\"summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"summary\",\"DEFAULT\":\"raw\"}"
@@ -87,18 +86,75 @@ secor_service_name:
     service_description: "Denormalized Events Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+  derived-denorm-events-backup:
+    consumer_group: "{{ env }}.summary.backup"
+    base_path: "telemetry-denormalized/summary"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.druid.events.summary"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Denormalized Summary Events Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"  
+  channel-telemetry-backup:
+    consumer_group: "{{ env }}.telemetry.channel.backup"
+    base_path: "data-exhaust/raw"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.telemetry.denorm"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: "eid"
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Data Exhaust Telemetry Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "50"
+    message_channel_identifier: "derivedlocationdata.state"
+    message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"
+  channel-summary-backup:
+    consumer_group: "{{ env }}.summary.channel.backup"
+    base_path: "data-exhaust/summary"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.druid.events.summary"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Data Exhaust Summary Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "50"
+    message_channel_identifier: "derivedlocationdata.state"
+    message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"    
   extractor-failed-backup:
     consumer_group: "{{ env }}.extractor.failed.backup"
     base_path: "extractor-failed"
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.extractor.failed"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "21600"
-    ostrich_port: "9989"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -106,6 +162,7 @@ secor_service_name:
     service_description: "Extractor failed events backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   assess-raw-events-backup:
     consumer_group: "{{ env }}.telemetry.assess.raw"
@@ -113,11 +170,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.assess.raw"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "21600"
-    ostrich_port: "9986"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -125,6 +181,7 @@ secor_service_name:
     service_description: "Telemetry raw assess events backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   telemetry-ingest-backup:
     consumer_group: "{{ env }}.telemetry.ingest.backup"
@@ -132,11 +189,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.ingest"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9985"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -144,6 +200,7 @@ secor_service_name:
     service_description: "Ingestion cluster Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "54"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   telemetry-duplicate-backup:
     consumer_group: "{{ env }}.telemetry.duplicate.backup"
@@ -151,11 +208,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.duplicate"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9984"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -163,6 +219,7 @@ secor_service_name:
     service_description: "Telemetry Duplicate Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   extractor-duplicate-backup:
     consumer_group: "{{ env }}.extractor.duplicate.backup"
@@ -170,11 +227,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.extractor.duplicate"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9983"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -182,6 +238,7 @@ secor_service_name:
     service_description: "Telemetry Duplicate Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   derived-telemetry-backup:
     consumer_group: "{{ env }}.telemetry.derived.unique.backup"
@@ -189,11 +246,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.telemetry.derived.unique"
-    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(':9092,')}}:9092"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
     zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
     max_file_size: "100000000"
     max_file_age: "600"
-    ostrich_port: "9982"
     partition_prefix_enabled: "true"
     partition_prefix_key: "eid"
     partition_prefix_mapping: "{\"ME_SESSION_SUMMARY\":\"session_summary\",\"ME_WORKFLOW_SUMMARY\":\"workflow_summary\",\"ME_DEIVCE_SUMMARY\":\"device_summary\",\"ME_ITEM_SUMMARY\":\"item_summary\",\"ME_WORKFLOW_USAGE_SUMMARY\":\"workflow_usage_summary\",\"ME_ITEM_USAGE_SUMMARY\":\"item_usage_summary\",\"ME_USAGE_SUMMARY\":\"usage_summary\",\"ME_DIALCODE_USAGE_SUMMARY\":\"dialcode_usage_summary\",\"DEFAULT\":\"me\"}"
@@ -201,6 +257,7 @@ secor_service_name:
     service_description: "Derived Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "52"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   device-profile-backup:
     consumer_group: "{{ env }}.events.device.profile.backup"
@@ -208,11 +265,10 @@ secor_service_name:
     timestamp_key: "updated_date"
     fallback_timestamp_key: "updated_date"
     topic: "{{ env }}.events.deviceprofile"
-    kafka_broker_host: "{{ ingestion_kafka_brokers }}"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
     zookeeper_quorum: "{{ ingestion_zookeepers }}"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9981"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -220,6 +276,7 @@ secor_service_name:
     service_description: "Device Profile Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   learning-events-backup:
     consumer_group: "{{ env }}.learning.graph.events.backup"
@@ -227,11 +284,10 @@ secor_service_name:
     timestamp_key: "createdOn"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ env }}.learning.graph.events"
-    kafka_broker_host: "{{ ingestion_kafka_brokers }}"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
     zookeeper_quorum: "{{ ingestion_zookeepers }}"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9980"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -239,6 +295,7 @@ secor_service_name:
     service_description: "Learning Events Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "56"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   learning-failed-backup:
     consumer_group: "{{ env }}.failed.learning.events.backup"
@@ -249,11 +306,10 @@ secor_service_name:
     container_name: "{{ secor.azure.container_name }}"
     azure_account_key: "{{ secor.azure.account_key }}"
     topic: "{{ env }}.learning.events.failed"
-    kafka_broker_host: "{{ ingestion_kafka_brokers }}"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
     zookeeper_quorum: "{{ ingestion_zookeepers }}"
     max_file_size: "100000000"
     max_file_age: "3600"
-    ostrich_port: "9979"
     partition_prefix_enabled: "true"
     partition_prefix_key: "jobName"
     partition_prefix_mapping: "{\"publish-pipeline\":\"publish_pipeline\",\"composite-search-indexer\":\"cs_index\",\"DEFAULT\":\"failed_events\"}"
@@ -261,6 +317,7 @@ secor_service_name:
     service_description: "Learning Failed Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "52"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   assess-events-backup:
     consumer_group: "{{ env }}.telemetry.assess.events.backup"
@@ -268,11 +325,10 @@ secor_service_name:
     timestamp_key: "assessmentTs"
     fallback_timestamp_key: "assessmentTs"
     topic: "{{ env }}.telemetry.assess"
-    kafka_broker_host: "{{ ingestion_kafka_brokers }}"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
     zookeeper_quorum: "{{ ingestion_zookeepers }}"
     max_file_size: "100000000"
     max_file_age: "21600"
-    ostrich_port: "9978"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -280,6 +336,7 @@ secor_service_name:
     service_description: "Telemetry Batch assess events backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "50"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
   ingestion-cluster-telemetry-backup:
     consumer_group: "{{ env }}.telemetry.ingestion.events.backup"
@@ -287,11 +344,10 @@ secor_service_name:
     timestamp_key: "syncts"
     fallback_timestamp_key: "@timestamp"
     topic: "{{ telemetry_ingestion_topic }}"
-    kafka_broker_host: "{{ ingestion_kafka_brokers }}"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
     zookeeper_quorum: "{{ ingestion_zookeepers }}"
     max_file_size: "100000000"
     max_file_age: "14400"
-    ostrich_port: "9977"
     partition_prefix_enabled: "false"
     partition_prefix_key: ""
     partition_prefix_mapping: "{}"
@@ -299,4 +355,62 @@ secor_service_name:
     service_description: "Ingestion cluster Telemetry Backup"
     service_monitor_schedule_hr: "23"
     service_monitor_schedule_min: "54"
+    message_channel_identifier: ""
     message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+  content-consumption-events-backup:
+    consumer_group: "{{ env }}.coursebatch.job.request.backup"
+    base_path: "content-consumption-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.coursebatch.job.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Content Consumption Events Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "56"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+  issue-certificate-events-backup:
+    consumer_group: "{{ env }}.issue.certificate.request.backup"
+    base_path: "issue-certificate-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.issue.certificate.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Issue Certificate Events Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "55"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+  content-auto-creation-events-backup:
+    consumer_group: "{{ env }}.auto.creation.job.request.backup"
+    base_path: "content-auto-creation-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env }}.auto.creation.job.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: "100000000"
+    max_file_age: "14400"
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    service_description: "Content Auto Creation Events Backup"
+    service_monitor_schedule_hr: "23"
+    service_monitor_schedule_min: "57"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
\ No newline at end of file
diff --git a/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml b/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml
index cc8c316a13..21fb631358 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml
+++ b/ansible/roles/secor-telemetry-backup-deploy/tasks/main.yml
@@ -25,7 +25,7 @@
 
 - name: Unarchive secor libraries
   unarchive:
-    src: "{{ analytics_user_home }}/secor-0.25-SNAPSHOT-bin.tar.gz"
+    src: "{{ analytics_user_home }}/secor-{{ secor.artifact_ver }}-bin.tar.gz"
     dest: "/mount/{{ item.key }}"
     copy: no
     owner: "{{ analytics_user }}"
diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2
index c0f76c6d1f..6a001efee8 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2
+++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.common.j2
@@ -165,7 +165,7 @@ kafka.fetch.min.bytes=
 kafka.fetch.wait.max.ms=
 
 # Port of the broker serving topic partition metadata.
-kafka.seed.broker.port=9092
+#kafka.seed.broker.port=9092
 
 # Zookeeper path at which kafka is registered. In Zookeeper parlance, this is referred
 # to as the chroot.
@@ -420,3 +420,76 @@ parquet.validation=false
 secor.orc.message.schema.*=struct<a:int\,b:int\,c:struct<d:int\,e:string>\,f:array<string>\,g:int>
 # Below config used for defining ORC schema provider class name. User can use the custom implementation for orc schema provider
 secor.orc.schema.provider=com.pinterest.secor.util.orc.schema.DefaultORCSchemaProvider
+
+# Changes related to 0.29 upgrade
+
+# If true, runs a final upload when the JVM is shut down (eg, from SIGTERM).
+# You may want to ensure that whatever system you use to run secor sets an
+# appropriate grace period to allow a full upload before a forced termination.
+secor.upload.on.shutdown=false
+
+# If true, uploads lastSeen offset onto ZK, see https://github.com/pinterest/secor/issues/600 for details
+secor.upload.secor.upload.last.seen.offset=false
+
+# If true, uploads are entirely deterministic, which can avoid some race conditions
+# which can lead to messages being backed up multiple times. This is incompatible with
+# secor.upload.on.shutdown=true, and ignores the values of secor.max.file.size.bytes,
+# secor.max.file.age.seconds, and secor.upload.minute_mark.
+#
+# In deterministic mode, you must set one or both of secor.max.file.timestamp.range.millis and
+# secor.max.input.payload.size.bytes. These determine when to upload, and are ignored outside
+# of deterministic mode.
+secor.upload.deterministic=false
+
+# Classname for the message iterator you want to use. The message iterator determines what kind of consumer
+# secor will use to communicate with kafka. com.pinterest.secor.reader.LegacyKafkaMessageIterator uses
+# the old kafka consumer written scala. Its not recommended to use the legacy iterator with kafka version >= 1.0 since it
+# does not support the new broker protocols. You may face significant performance degradation on your brokers if you use it
+kafka.message.iterator.className=com.pinterest.secor.reader.SecorKafkaMessageIterator
+
+# Classname for the kafka client used by utility classes like PartitionFinalizer and Progress monitor.
+# Legacy client is kept for compatibility purposes and is deprecated
+kafka.client.className=com.pinterest.secor.common.SecorKafkaClient
+
+
+########################################
+# Optional: Kafka New Consumer Configs #
+########################################
+
+# Same as old configuration. Except accepted values are earliest and latest instead of smallest
+# and largest
+kafka.new.consumer.auto.offset.reset=earliest
+
+# Comma-separated list of topics to consume. Please note that this is not a regular expression.
+# If that's what you want, you can use "secor.kafka.topic_filter" instead.
+kafka.new.consumer.topic.list={{ secor_service_name[item[0]].topic }}
+
+kafka.new.consumer.poll.timeout.seconds=10
+kafka.new.consumer.request.timeout.ms=10000
+kafka.new.consumer.ssl.key.password=
+kafka.new.consumer.ssl.keystore.location=
+kafka.new.consumer.ssl.keystore.password=
+kafka.new.consumer.ssl.truststore.location=
+kafka.new.consumer.ssl.truststore.password=
+kafka.new.consumer.isolation.level=
+kafka.new.consumer.max.poll.interval.ms=
+kafka.new.consumer.max.poll.records=
+kafka.new.consumer.sasl.client.callback.handler.class=
+kafka.new.consumer.sasl.jaas.config=
+kafka.new.consumer.sasl.kerberos.service.name=
+kafka.new.consumer.sasl.login.callback.handler.class=
+kafka.new.consumer.sasl.login.class=
+kafka.new.consumer.sasl.mechanism=
+kafka.new.consumer.security.protocol=
+kafka.new.consumer.ssl.enabled.protocols=
+kafka.new.consumer.ssl.keystore.type=
+kafka.new.consumer.ssl.protocol=
+kafka.new.consumer.ssl.provider=
+kafka.new.consumer.ssl.truststore.type=
+kafka.new.consumer.partition.assignment.strategy.class=
+
+########################################
+# End: Kafka New Consumer Configs      #
+########################################
+
+kafka.fetch.max.bytes=
\ No newline at end of file
diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2
index eb243d293a..38f7b95ca3 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2
+++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.j2
@@ -29,7 +29,7 @@ secor.swift.container=logsContainer
 ################
 
 kafka.seed.broker.host={{ secor_service_name[item[0]].kafka_broker_host }}
-kafka.seed.broker.port=
+kafka.seed.broker.port=9092
 
 zookeeper.quorum={{ secor_service_name[item[0]].zookeeper_quorum }}
 
diff --git a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2 b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2
index a9f2247aa5..b6d06e14ef 100644
--- a/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2
+++ b/ansible/roles/secor-telemetry-backup-deploy/templates/secor.partition.j2
@@ -35,7 +35,7 @@ secor.azure.path={{ secor_service_name[item[0]].base_path }}
 secor.local.path=/mount/{{ item.0 }}/message_logs/partition
 
 # Port of the Ostrich server.
-ostrich.port={{ secor_service_name[item[0]].ostrich_port }}
+#ostrich.port=
 
 # Secor custom properties
 
@@ -63,3 +63,5 @@ secor.max.file.age.policy=oldest
 # currentDate - Time of upload in YYYYMMDD format
 # folder - Folder to use based on message id map lookup
 secor.s3.output_file_pattern={{ secor_service_name[item[0]].output_file_pattern }}
+
+secor.partition.message.channel.identifier={{ secor_service_name[item[0]].message_channel_identifier }}
diff --git a/ansible/roles/setup-kafka/defaults/main.yml b/ansible/roles/setup-kafka/defaults/main.yml
index 08e13e4038..ddcdd829aa 100644
--- a/ansible/roles/setup-kafka/defaults/main.yml
+++ b/ansible/roles/setup-kafka/defaults/main.yml
@@ -3,271 +3,214 @@ kafka_partition_override_size: 5242880
 dp_downstream_kafka_max_message_bytes: 1572864
 
 ingestion_kafka_topics:
-  - name: telemetry.ingest
-    num_of_partitions: 4
+  - name: telemetry.ingestion
+    num_of_partitions: 2
+    replication_factor: 1
+  - name: events.deviceprofile
+    num_of_partitions: 2
     replication_factor: 1
 
 ingestion_kafka_overriden_topics:
-  - name: telemetry.ingest
-    retention_time: 604800000
+  - name: telemetry.ingestion
+    retention_time: 259200000
     replication_factor: 1
     max_message_bytes: 5242880
+  - name: events.deviceprofile
+    retention_time: 172800000
+    replication_factor: 1
 
 processing_kafka_overriden_topics:
-  - name: telemetry.raw
+  - name: analytics.job_queue
     retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.valid
-    retention_time: 21600000
-    replication_factor: 1
-  - name: telemetry.unique
+  - name: analytics_metrics
     retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.duplicate
-    retention_time: 21600000
-    replication_factor: 1
-  - name: telemetry.sink
-    retention_time: 21600000
-    replication_factor: 1
-  - name: telemetry.with_location
-    retention_time: 86400000
-    replication_factor: 1
-  - name: telemetry.audit
-    retention_time: 86400000
-    replication_factor: 1
-  - name: telemetry.denorm
-    retention_time: 86400000
-    replication_factor: 1
-  - name: telemetry.denorm.valid
-    retention_time: 86400000
-    replication_factor: 1
-  - name: telemetry.de_normalized
-    retention_time: 21600000
+  - name: druid.events.error
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.log
-    retention_time: 21600000
+  - name: druid.events.log
+    retention_time: 172800000
     replication_factor: 1
-  - name: analytics.job_queue
-    retention_time: 28800000
+  - name: druid.events.summary
+    retention_time: 172800000
     replication_factor: 1
-  - name: learning.graph.events
-    retention_time: 86400000
+  - name: druid.events.telemetry
+    retention_time: 172800000
     replication_factor: 1
-  - name: pipeline_metrics
-    retention_time: 86400000
+  - name: events.deviceprofile
+    retention_time: 172800000
     replication_factor: 1
   - name: prom.monitoring.metrics
-    retention_time: 604800000
-    replication_factor: 1
-  - name: metrics
-    retention_time: 604800000
+    retention_time: 172800000
     replication_factor: 1
-  - name: learning.graph.events.fail
-    retention_time: 1296000000
+  - name: telemetry.assess
+    retention_time: 172800000
     replication_factor: 1
-  - name: analytics_metrics
-    retention_time: 21600000
+  - name: telemetry.assess.failed
+    retention_time: 172800000
     replication_factor: 1
-  - name: learning.job.request
-    retention_time: 86400000
+  - name: telemetry.assess.raw
+    retention_time: 172800000
     replication_factor: 1
-  - name: learning.job.request.fail
-    retention_time: 1296000000
+  - name: telemetry.audit
+    retention_time: 172800000
     replication_factor: 1
-  - name: learning.republish.job.request
-    retention_time: 1296000000
+  - name: telemetry.denorm 
+    retention_time: 172800000
     replication_factor: 1
   - name: telemetry.derived
-    retention_time: 21600000
-    replication_factor: 1
-  - name: events.telemetry
-    retention_time: 86400000
+    retention_time: 172800000
     replication_factor: 1
-  - name: events.summary
-    retention_time: 86400000
+  - name: telemetry.derived.unique
+    retention_time: 172800000
     replication_factor: 1
-  - name: events.log
-    retention_time: 86400000
-  - name: events.error
-    retention_time: 86400000
-  - name: telemetry.assess
-    retention_time: 604800000
+  - name: telemetry.duplicate
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.assess.redact
-    retention_time: 86400000
+  - name: telemetry.error
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.assess.raw
-    retention_time: 86400000
+  - name: telemetry.extractor.duplicate
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.derived.unique
-    retention_time: 21600000
+  - name: telemetry.extractor.failed
+    retention_time: 172800000
     replication_factor: 1
-  # Error/Failed Topics
   - name: telemetry.failed
-    retention_time: 86400000
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.malformed
-    retention_time: 21600000
+  - name: telemetry.ingest
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.extractor.failed
-    retention_time: 21600000
+  - name: telemetry.metrics
+    retention_time: 172800000
     replication_factor: 1
-  - name: events.deviceprofile
-    retention_time: 604800000
+  - name: telemetry.raw
+    retention_time: 172800000
     replication_factor: 1
-  - name: system.command
-    retention_time: 3600
+  - name: telemetry.unique
+    retention_time: 172800000
     replication_factor: 1
-  - name: lms.audit.events
+  - name: telemetry.unique.latest
     retention_time: 172800000
     replication_factor: 1
-  - name: lms.sso.events
+  - name: telemetry.unique.primary
     retention_time: 172800000
     replication_factor: 1
-  - name: lms.user.account.merge
+  - name: telemetry.unique.secondary
     retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.metrics
-    retention_time: 604800000
+  - name: ml.observation.raw
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.assess.failed
-    retention_time: 2592000000
+  - name: ml.observation.druid
+    retention_time: 172800000
     replication_factor: 1
-  - name: telemetry.share
-    retention_time: 2592000000
-    replication_factor: 1        
 
 processing_kafka_topics:
-  - name: system.command
-    retention_time: 3600
+  - name: analytics.job_queue
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.raw
-    num_of_partitions: 4
-    replication_factor: 1
-  - name: telemetry.valid
-    num_of_partitions: 4
-    replication_factor: 1
-  - name: telemetry.unique
-    num_of_partitions: 8
-    replication_factor: 1
-  - name: telemetry.duplicate
+  - name: analytics_metrics
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.sink
-    num_of_partitions: 8
-    replication_factor: 1
-  - name: telemetry.with_location
-    num_of_partitions: 8
-    replication_factor: 1
-  - name: telemetry.audit
+  - name: druid.events.error
     num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.denorm
-    num_of_partitions: 4
-    replication_factor: 1
-  - name: telemetry.denorm.valid
+  - name: druid.events.log
     num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.de_normalized
-    num_of_partitions: 4
-    replication_factor: 1
-  - name: events.telemetry
-    num_of_partitions: 4
-    replication_factor: 1
-  - name: events.summary
-    num_of_partitions: 4
+  - name: druid.events.summary
+    num_of_partitions: 2
     replication_factor: 1
-  - name: events.log
+  - name: druid.events.telemetry
     num_of_partitions: 2
     replication_factor: 1
-  - name: events.error
+  - name: events.deviceprofile
     num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.log
+  - name: prom.monitoring.metrics
     num_of_partitions: 1
     replication_factor: 1
-  - name: analytics.job_queue
-    num_of_partitions: 1
+  - name: telemetry.assess
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.graph.events
-    num_of_partitions: 1
+  - name: telemetry.assess.failed
+    num_of_partitions: 2
     replication_factor: 1
-  - name: pipeline_metrics
-    num_of_partitions: 1
+  - name: telemetry.assess.raw
+    num_of_partitions: 2
     replication_factor: 1
-  - name: metrics
-    num_of_partitions: 1
+  - name: telemetry.audit
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.graph.events.fail
-    num_of_partitions: 1
+  - name: telemetry.denorm 
+    num_of_partitions: 2
     replication_factor: 1
-  - name: analytics_metrics
-    num_of_partitions: 1
+  - name: telemetry.derived
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.job.request 
-    num_of_partitions: 1
+  - name: telemetry.derived.unique
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.job.request.fail
-    num_of_partitions: 1
+  - name: telemetry.duplicate
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.republish.job.request
+  - name: telemetry.error
     num_of_partitions: 1
     replication_factor: 1
-  - name: qrimage.request
-    num_of_partitions: 1
+  - name: telemetry.extractor.duplicate
+    num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.derived
-    num_of_partitions: 1
+  - name: telemetry.extractor.failed
+    num_of_partitions: 2
     replication_factor: 1
   - name: telemetry.failed
-    num_of_partitions: 1
-    replication_factor: 1
-  - name: telemetry.malformed
-    num_of_partitions: 1
+    num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.extractor.failed
-    num_of_partitions: 1
+  - name: telemetry.ingest
+    num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.indexer.failed
-    num_of_partitions: 1
+  - name: telemetry.metrics
+    num_of_partitions: 2
     replication_factor: 1
-  - name: learning.events.failed
-    num_of_partitions: 1
+  - name: telemetry.raw
+    num_of_partitions: 2
     replication_factor: 1
-  - name: events.deviceprofile
-    num_of_partitions: 1
+  - name: telemetry.unique
+    num_of_partitions: 2
     replication_factor: 1
-  - name: lms.audit.events
-    num_of_partitions: 1
+  - name: telemetry.unique.latest
+    num_of_partitions: 2
     replication_factor: 1
-  - name: lms.sso.events
-    num_of_partitions: 4
+  - name: telemetry.unique.primary
+    num_of_partitions: 2
     replication_factor: 1
-  - name: lms.user.account.merge
-    num_of_partitions: 1
+  - name: telemetry.unique.secondary
+    num_of_partitions: 2
     replication_factor: 1
-  - name: telemetry.assess
+  - name: druid.events.log
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.assess.redact
+  - name: druid.events.telemetry
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.assess.raw
+  - name: druid.events.summary
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.derived.unique
+  - name: ml.observation.raw
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.metrics
+  - name: ml.observation.druid
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.assess.failed
+  - name: qrimage.request
     num_of_partitions: 1
     replication_factor: 1
-  - name: telemetry.share
+  - name: events.telemetry
     num_of_partitions: 1
     replication_factor: 1
-  - name: prom.monitoring.metrics
+  - name: events.summary
     num_of_partitions: 1
     replication_factor: 1
diff --git a/ansible/roles/setup-kafka/tasks/main.yml b/ansible/roles/setup-kafka/tasks/main.yml
index 1e34906e90..1f73a57ca4 100644
--- a/ansible/roles/setup-kafka/tasks/main.yml
+++ b/ansible/roles/setup-kafka/tasks/main.yml
@@ -1,5 +1,5 @@
 - name: create topics
-  command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }}
+  command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{ingestion_zookeeper_ip}}:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }}
   with_items: "{{ingestion_kafka_topics}}"
   ignore_errors: true
   when: kafka_id=="1"
@@ -7,14 +7,14 @@
     - ingestion-kafka
 
 - name: override retention time
-  command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }}
+  command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{ingestion_zookeeper_ip}}:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }}
   with_items: "{{ingestion_kafka_overriden_topics}}"
   when: kafka_id=="1" and item.retention_time is defined
   tags:
     - ingestion-kafka
 
 - name: create topics
-  command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }}
+  command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --create --topic {{env}}.{{item.name}} --partitions {{ item.num_of_partitions }} --replication-factor {{ item.replication_factor }}
   with_items: "{{processing_kafka_topics}}"
   ignore_errors: true
   when: kafka_id=="1"
@@ -22,29 +22,36 @@
     - processing-kafka
 
 - name: override retention time
-  command: /opt/kafka/bin/kafka-topics.sh --zookeeper localhost:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }}
+  command: /opt/kafka/bin/kafka-topics.sh --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{env}}.{{item.name}} --config retention.ms={{ item.retention_time }}
   with_items: "{{processing_kafka_overriden_topics}}"
   when: kafka_id=="1" and item.retention_time is defined  
   tags:
     - processing-kafka
 
 - name: overriding default partition size
-  shell: "/opt/kafka_2.12-1.1.0/bin/kafka-topics.sh  --zookeeper localhost:2181 --alter --topic {{item}} --config max.message.bytes={{kafka_partition_override_size}}"
+  shell: "/opt/kafka/bin/kafka-topics.sh  --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{item}} --config max.message.bytes={{kafka_partition_override_size}}"
   with_items:
     - "{{env}}.telemetry.ingest"
     - "{{env}}.telemetry.extractor.failed"
 
 - name: overriding data pipeline downstream topic size
-  shell: "/opt/kafka/bin/kafka-topics.sh  --zookeeper localhost:2181 --alter --topic {{item}} --config max.message.bytes={{dp_downstream_kafka_max_message_bytes}}"
+  shell: "/opt/kafka/bin/kafka-topics.sh  --zookeeper {{processing_zookeeper_ip}}:2181 --alter --topic {{item}} --config max.message.bytes={{dp_downstream_kafka_max_message_bytes}}"
   with_items:
-    - "{{env}}.telemetry.valid"
-    - "{{env}}.telemetry.unique"
-    - "{{env}}.telemetry.duplicate"
-    - "{{env}}.telemetry.sink"
-    - "{{env}}.telemetry.with_location"
+    - "{{env}}.druid.events.error"
+    - "{{env}}.druid.events.log"
+    - "{{env}}.druid.events.summary"
+    - "{{env}}.druid.events.telemetry"
+    - "{{env}}.telemetry.assess.raw"
+    - "{{env}}.telemetry.audit"
     - "{{env}}.telemetry.denorm"
-    - "{{env}}.telemetry.denorm.valid"
-    - "{{env}}.telemetry.log"
-    - "{{env}}.events.telemetry"
-    - "{{env}}.events.summary"
-    - "{{env}}.events.log"
+    - "{{env}}.telemetry.duplicate"
+    - "{{env}}.telemetry.error"
+    - "{{env}}.telemetry.extractor.duplicate"
+    - "{{env}}.telemetry.extractor.failed"
+    - "{{env}}.telemetry.failed"
+    - "{{env}}.telemetry.ingest"
+    - "{{env}}.telemetry.raw"
+    - "{{env}}.telemetry.unique"
+    - "{{env}}.telemetry.unique.latest"
+    - "{{env}}.telemetry.unique.primary"
+    - "{{env}}.telemetry.unique.secondary"
diff --git a/ansible/roles/yarn/defaults/main.yml b/ansible/roles/yarn/defaults/main.yml
deleted file mode 100644
index 4060075e8e..0000000000
--- a/ansible/roles/yarn/defaults/main.yml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-yarn_deploy_dir: /home/ecosystem/.deploy
-repo_folder: /home/hduser/Ecosystem-Platform
-hadoop_tarball: hadoop-{{hadoop_version}}.tar.gz
-hadoop_download_url: https://archive.apache.org/dist/hadoop/common/hadoop-{{hadoop_version}}/{{hadoop_tarball}}
-scala_tarball: scala-{{scala_version}}.tgz
-scala_download_url: http://www.scala-lang.org/files/archive/{{scala_tarball}}
-hadoop_yarn_home: /usr/local/hadoop-{{hadoop_version}}
-hadoop_version: 2.7.2
-scala_version: 2.10.4
-
-yarn_config_override: true
-yarn_vmem_check_enabled: false
-yarn_vmem_pmem_ratio: 2.1
-yarn_vcores: 16
-yarn_resource_memory: 20000
diff --git a/ansible/roles/yarn/files/truncate_logs.sh b/ansible/roles/yarn/files/truncate_logs.sh
deleted file mode 100644
index 7dddd9d702..0000000000
--- a/ansible/roles/yarn/files/truncate_logs.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-# Truncate hadoop/yarn userlogs and keep the last 100 lines
-
-HADOOP_LOGS_HOME=/usr/local/hadoop/logs/userlogs
-
-for d in $HADOOP_LOGS_HOME/*/*/ ; do (cd $d && tail -n 100 stdout > stdout.tmp && cat stdout.tmp > stdout && rm stdout.tmp); done
-
-LOGSTASH_LOGS=/var/log/logstash
-tail -n 100 $LOGSTASH_LOGS/logstash.stdout > $LOGSTASH_LOGS/logstash.stdout.tmp
-cat $LOGSTASH_LOGS/logstash.stdout.tmp > $LOGSTASH_LOGS/logstash.stdout
-rm $LOGSTASH_LOGS/logstash.stdout.tmp
-
-HADOOP_TMP_USERLOGS=/usr/local/hadoop/logs/userlogs
-
-for g in $HADOOP_TMP_USERLOGS/*/*/ do
-	cd $g
-	tail -n 100 stdout > stdout.tmp 
-	cat stdout.tmp > stdout
-	rm stdout.tmp
-done
diff --git a/ansible/roles/yarn/tasks/common.yml b/ansible/roles/yarn/tasks/common.yml
deleted file mode 100644
index 343aa54274..0000000000
--- a/ansible/roles/yarn/tasks/common.yml
+++ /dev/null
@@ -1,76 +0,0 @@
-- name: Common tasks for yarn master and slave
-  block:
-  - name: Download and extract hadoop tarball
-    unarchive:
-      src: "{{hadoop_download_url}}"
-      dest: "/usr/local/"
-      owner: hduser
-      group: hadoop
-      creates: "{{hadoop_yarn_home}}"
-      remote_src: yes
-
-  - name: Creates symlink
-    file:
-      src: /usr/local/hadoop-{{hadoop_version}}
-      dest: /usr/local/hadoop
-      owner: hduser
-      group: hadoop
-      state: link
-
-  - name: creating conf dir
-    file:
-      path: "{{hadoop_yarn_home}}/conf"
-      owner: hduser
-      group: hadoop
-      recurse: yes
-      state: directory
-
-  - name: Templating configs
-    template:
-      src: "{{item}}"
-      dest: "{{hadoop_yarn_home}}/conf/{{item}}"
-      owner: hduser
-      group: hadoop
-    with_items:
-      - yarn-site.xml
-      - capacity-scheduler.xml
-      - core-site.xml
-      - log4j.properties
-      - hadoop-env.sh
-
-  - name: Downloading artifacts
-    get_url:
-      url: "http://search.maven.org/remotecontent?filepath=org/{{item}}"
-      dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
-      owner: hduser
-      group: hadoop
-    with_items:
-      - clapper/grizzled-slf4j_2.10/1.0.1/grizzled-slf4j_2.10-1.0.1.jar
-      - apache/samza/samza-yarn_2.10/0.8.0/samza-yarn_2.10-0.8.0.jar
-      - apache/samza/samza-core_2.10/0.8.0/samza-core_2.10-0.8.0.jar
-
-  - name: Download and extract scala
-    unarchive:
-      src: "{{scala_download_url}}"
-      dest: "/usr/local/"
-      owner: hduser
-      group: hadoop
-      remote_src: yes
-
-  - name: Creates symlink
-    file:
-      src: "/usr/local/scala-{{scala_version}}"
-      dest: /usr/local/scala
-      owner: hduser
-      group: hadoop
-      state: link
-
-  - name: copying scala files
-    copy:
-      src: "/usr/local/scala-{{scala_version}}/lib/{{item}}"
-      dest: "{{hadoop_yarn_home}}/share/hadoop/hdfs/lib/"
-      remote_src: true
-    with_items:
-      - scala-compiler.jar
-      - scala-library.jar
-  delegate_to: "{{slave|default(inventory_hostname)}}"
diff --git a/ansible/roles/yarn/tasks/main.yml b/ansible/roles/yarn/tasks/main.yml
deleted file mode 100644
index a297e9a951..0000000000
--- a/ansible/roles/yarn/tasks/main.yml
+++ /dev/null
@@ -1,55 +0,0 @@
-- name: Debian | Install Maven
-  apt:
-    pkg: "{{item}}"
-    update_cache: yes
-    state: latest
-    install_recommends: yes
-  with_items:
-    - maven
-    - git
-
-# Running common tasks in master
-- include: common.yml
-
-- lineinfile:
-    dest: /home/hduser/.bashrc
-    state: present
-    regexp: '^HADOOP_YARN_HOME'
-    line: 'HADOOP_YARN_HOME={{hadoop_yarn_home}}'
-
-- lineinfile:
-    dest: /home/hduser/.bashrc
-    state: present
-    regexp: '^HADOOP_CONF_DIR'
-    line: 'HADOOP_CONF_DIR=$HADOOP_YARN_HOME/conf'
-
-- file:
-    path: "/{{hadoop_yarn_home}}/conf/slaves"
-    state: touch
-
-- lineinfile:
-    dest: "/{{hadoop_yarn_home}}/conf/slaves"
-    state: present
-    regexp: "^{{item}}"
-    line: "{{item}}"
-  with_items: "{{yarn_slaves}}"
-
-# Running common tasks in slaves
-- name: Running common tasks in slaves
-  include: common.yml
-  with_items: "{{yarn_slaves}}"
-  loop_control:
-    loop_var: slave
-
-- name: Copy truncate_files.sh
-  copy:
-    src: truncate_logs.sh
-    dest: /usr/local/bin
-    mode: 755
-
-- name: Add truncate logs to cron
-  cron:
-    name: "Truncate yarn logs"
-    minute: "0"
-    job: "/usr/local/bin/truncate_logs.sh"
-    backup: yes
diff --git a/ansible/roles/yarn/templates/capacity-scheduler.xml b/ansible/roles/yarn/templates/capacity-scheduler.xml
deleted file mode 100644
index 4161b7ac03..0000000000
--- a/ansible/roles/yarn/templates/capacity-scheduler.xml
+++ /dev/null
@@ -1,111 +0,0 @@
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
-
-  <property>
-    <name>yarn.scheduler.capacity.maximum-applications</name>
-    <value>10000</value>
-    <description>
-      Maximum number of applications that can be pending and running.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.maximum-am-resource-percent</name>
-    <value>0.5</value>
-    <description>
-      Maximum percent of resources in the cluster which can be used to run
-      application masters i.e. controls number of concurrent running
-      applications.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.resource-calculator</name>
-    <value>org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator</value>
-    <description>
-      The ResourceCalculator implementation to be used to compare
-      Resources in the scheduler.
-      The default i.e. DefaultResourceCalculator only uses Memory while
-      DominantResourceCalculator uses dominant-resource to compare
-      multi-dimensional resources such as Memory, CPU etc.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.queues</name>
-    <value>default</value>
-    <description>
-      The queues at the this level (root is the root queue).
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.capacity</name>
-    <value>100</value>
-    <description>Default queue target capacity.</description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.user-limit-factor</name>
-    <value>1</value>
-    <description>
-      Default queue user limit a percentage from 0.0 to 1.0.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.maximum-capacity</name>
-    <value>100</value>
-    <description>
-      The maximum capacity of the default queue.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.state</name>
-    <value>RUNNING</value>
-    <description>
-      The state of the default queue. State can be one of RUNNING or STOPPED.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.acl_submit_applications</name>
-    <value>*</value>
-    <description>
-      The ACL of who can submit jobs to the default queue.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.root.default.acl_administer_queue</name>
-    <value>*</value>
-    <description>
-      The ACL of who can administer jobs on the default queue.
-    </description>
-  </property>
-
-  <property>
-    <name>yarn.scheduler.capacity.node-locality-delay</name>
-    <value>-1</value>
-    <description>
-      Number of missed scheduling opportunities after which the CapacityScheduler
-      attempts to schedule rack-local containers.
-      Typically this should be set to number of racks in the cluster, this
-      feature is disabled by default, set to -1.
-    </description>
-  </property>
-
-</configuration>
diff --git a/ansible/roles/yarn/templates/config.j2 b/ansible/roles/yarn/templates/config.j2
deleted file mode 100644
index f1b411ed5f..0000000000
--- a/ansible/roles/yarn/templates/config.j2
+++ /dev/null
@@ -1,3 +0,0 @@
-Host {{item}}
-  IdentityFile ~/.ssh/hadoop_rsa
-  StrictHostKeyChecking no
diff --git a/ansible/roles/yarn/templates/core-site.xml b/ansible/roles/yarn/templates/core-site.xml
deleted file mode 100644
index d5a0752d52..0000000000
--- a/ansible/roles/yarn/templates/core-site.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-    <property>
-      <name>fs.http.impl</name>
-      <value>org.apache.samza.util.hadoop.HttpFileSystem</value>
-    </property>
-</configuration>
diff --git a/ansible/roles/yarn/templates/hadoop-env.sh b/ansible/roles/yarn/templates/hadoop-env.sh
deleted file mode 100644
index fd200c373c..0000000000
--- a/ansible/roles/yarn/templates/hadoop-env.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#export JAVA_HOME=/usr/lib/jvm/java-8-oracle
-export JAVA_HOME=/opt/jdk1.8.0_121
diff --git a/ansible/roles/yarn/templates/log4j.properties b/ansible/roles/yarn/templates/log4j.properties
deleted file mode 100644
index 4181560fcc..0000000000
--- a/ansible/roles/yarn/templates/log4j.properties
+++ /dev/null
@@ -1,267 +0,0 @@
-# Copyright 2011 The Apache Software Foundation
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Define some default values that can be overridden by system properties
-hadoop.root.logger=INFO,console
-hadoop.log.dir=.
-hadoop.log.file=hadoop.log
-
-# Define the root logger to the system property "hadoop.root.logger".
-log4j.rootLogger=${hadoop.root.logger}, EventCounter
-
-# Logging Threshold
-log4j.threshold=ALL
-
-# Null Appender
-log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
-
-#
-# Rolling File Appender - cap space usage at 5gb.
-#
-hadoop.log.maxfilesize=25MB
-hadoop.log.maxbackupindex=1
-log4j.appender.RFA=org.apache.log4j.RollingFileAppender
-log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize}
-log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex}
-
-log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# Daily Rolling File Appender
-#
-
-log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
-
-# Rollver at midnight
-log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
-
-# 30-day backup
-log4j.appender.DRFA.MaxBackupIndex=1
-log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
-
-# Pattern format: Date LogLevel LoggerName LogMessage
-log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-# Debugging Pattern format
-#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
-
-
-#
-# console
-# Add "console" to rootlogger above if you want to use this
-#
-
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-
-#
-# TaskLog Appender
-#
-
-#Default values
-hadoop.tasklog.taskid=null
-hadoop.tasklog.iscleanup=false
-hadoop.tasklog.noKeepSplits=4
-hadoop.tasklog.totalLogFileSize=100
-hadoop.tasklog.purgeLogSplits=true
-hadoop.tasklog.logsRetainHours=1
-
-log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
-log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
-log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
-log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
-
-log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
-log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-
-#
-# HDFS block state change log from block manager
-#
-# Uncomment the following to suppress normal block state change
-# messages from BlockManager in NameNode.
-#log4j.logger.BlockStateChange=WARN
-
-#
-#Security appender
-#
-hadoop.security.logger=INFO,NullAppender
-hadoop.security.log.maxfilesize=256MB
-hadoop.security.log.maxbackupindex=20
-log4j.category.SecurityLogger=${hadoop.security.logger}
-hadoop.security.log.file=SecurityAuth-${user.name}.audit
-log4j.appender.RFAS=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize}
-log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex}
-
-#
-# Daily Rolling Security appender
-#
-log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender
-log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file}
-log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout
-log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
-log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd
-
-#
-# hadoop configuration logging
-#
-
-# Uncomment the following line to turn off configuration deprecation warnings.
-# log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
-
-#
-# hdfs audit logging
-#
-hdfs.audit.logger=INFO,NullAppender
-hdfs.audit.log.maxfilesize=25MB
-hdfs.audit.log.maxbackupindex=1
-log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger}
-log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false
-log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log
-log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize}
-log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex}
-
-#
-# mapred audit logging
-#
-mapred.audit.logger=INFO,NullAppender
-mapred.audit.log.maxfilesize=256MB
-mapred.audit.log.maxbackupindex=20
-log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger}
-log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false
-log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender
-log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log
-log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout
-log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize}
-log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex}
-
-# Custom Logging levels
-
-#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
-#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
-#log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG
-
-# Jets3t library
-log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
-
-#
-# Event Counter Appender
-# Sends counts of logging messages at different severity levels to Hadoop Metrics.
-#
-log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
-
-#
-# Job Summary Appender
-#
-# Use following logger to send summary to separate file defined by
-# hadoop.mapreduce.jobsummary.log.file :
-# hadoop.mapreduce.jobsummary.logger=INFO,JSA
-#
-hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
-hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
-hadoop.mapreduce.jobsummary.log.maxfilesize=256MB
-hadoop.mapreduce.jobsummary.log.maxbackupindex=20
-log4j.appender.JSA=org.apache.log4j.RollingFileAppender
-log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file}
-log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize}
-log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex}
-log4j.appender.JSA.layout=org.apache.log4j.PatternLayout
-log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
-log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger}
-log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false
-
-#
-# Yarn ResourceManager Application Summary Log
-#
-# Set the ResourceManager summary log filename
-yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log
-# Set the ResourceManager summary log level and appender
-yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger}
-#yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY
-
-# To enable AppSummaryLogging for the RM,
-# set yarn.server.resourcemanager.appsummary.logger to
-# <LEVEL>,RMSUMMARY in hadoop-env.sh
-
-# Appender for ResourceManager Application Summary Log
-# Requires the following properties to be set
-#    - hadoop.log.dir (Hadoop Log directory)
-#    - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename)
-#    - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender)
-
-log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger}
-log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false
-log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender
-log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file}
-log4j.appender.RMSUMMARY.MaxFileSize=25MB
-log4j.appender.RMSUMMARY.MaxBackupIndex=1
-log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout
-log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-
-# HS audit log configs
-#mapreduce.hs.audit.logger=INFO,HSAUDIT
-#log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger}
-#log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false
-#log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender
-#log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log
-#log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout
-#log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n
-#log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd
-
-# Http Server Request Logs
-#log4j.logger.http.requests.namenode=INFO,namenoderequestlog
-#log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log
-#log4j.appender.namenoderequestlog.RetainDays=1
-
-#log4j.logger.http.requests.datanode=INFO,datanoderequestlog
-#log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log
-#log4j.appender.datanoderequestlog.RetainDays=3
-
-#log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog
-#log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log
-#log4j.appender.resourcemanagerrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog
-#log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log
-#log4j.appender.jobhistoryrequestlog.RetainDays=3
-
-#log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog
-#log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender
-#log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log
-#log4j.appender.nodemanagerrequestlog.RetainDays=3
diff --git a/ansible/roles/yarn/templates/yarn-site.xml b/ansible/roles/yarn/templates/yarn-site.xml
deleted file mode 100644
index 796f6450c3..0000000000
--- a/ansible/roles/yarn/templates/yarn-site.xml
+++ /dev/null
@@ -1,64 +0,0 @@
-<?xml version="1.0"?>
-<!--
-  Licensed under the Apache License, Version 2.0 (the "License");
-  you may not use this file except in compliance with the License.
-  You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-  Unless required by applicable law or agreed to in writing, software
-  distributed under the License is distributed on an "AS IS" BASIS,
-  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-  See the License for the specific language governing permissions and
-  limitations under the License. See accompanying LICENSE file.
--->
-<configuration>
-  <property>
-      <name>yarn.resourcemanager.hostname</name>
-      <!-- hostname that is accessible from all NMs -->
-      <value>{{resourcemanager}}</value>
-  </property>
-  <property>
-     <name>yarn.nodemanager.resource.memory-mb</name>
-     <value>{{yarn_resource_memory}}</value>
-  </property>
-  <property>
-    <name>yarn.scheduler.minimum-allocation-mb</name>
-    <value>128</value>
-  </property>
-  <property>
-    <name>mapreduce.job.userlog.retain.hours</name>
-    <value>240</value>
-  </property>
-  <property>
-    <name>yarn.log-aggregation-enable</name>
-    <value>false</value>
-  </property>
-  <property>
-    <name>yarn.nodemanager.log.retain-seconds</name>
-    <value>3600</value>
-  </property>
-    <property>
-    <name>yarn.nodemanager.recovery.enabled</name>
-    <value>true</value>
-  </property>
-  <property>
-      <name>yarn.nodemanager.address</name>
-    <value>0.0.0.0:45454</value>
-  </property>
-  <property>
-    <name>yarn.nodemanager.resource.cpu-vcores</name>
-    <value>{{yarn_vcores}}</value>
-  </property>
-  {% if yarn_config_override is defined %}
-  <property>
-   <name>yarn.nodemanager.vmem-check-enabled</name>
-   <value>{{yarn_vmem_check_enabled}}</value>
-  </property>
-  <property>
-   <name>yarn.nodemanager.vmem-pmem-ratio</name>
-   <value>{{yarn_vmem_pmem_ratio}}</value>
-  </property>
-  {% endif %}
-</configuration>
-
diff --git a/ansible/roles/zookeeper-upgrade/defaults/main.yml b/ansible/roles/zookeeper-upgrade/defaults/main.yml
index efb0a39a40..6425d3340f 100644
--- a/ansible/roles/zookeeper-upgrade/defaults/main.yml
+++ b/ansible/roles/zookeeper-upgrade/defaults/main.yml
@@ -1,6 +1,6 @@
 ---
-zookeeper_version: 3.4.12
-zookeeper_url: http://www.us.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/zookeeper-{{zookeeper_version}}.tar.gz
+zookeeper_version: 3.6.3
+zookeeper_url: "https://downloads.apache.org/zookeeper/stable/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz"
 zookeeper_port: 2181
 zk_jvm_opts: "-Xms128m -Xmx128m"
 verify: True
@@ -10,10 +10,13 @@ sync_limit: 2
 tick_time: 2000
 zoo_id: 1
 maxclinetconnection_limit: 200
-data_dir: /var/lib/zookeeper
+zookeeper_data_dir: /opt/zookeeper
 log_dir: /var/log/zookeeper
 client_port: "{{ zookeeper_port }}"
 zookeeper_group: "{{ groups['zookeeper'] }}"
+zookeeper_user: zookeeper
+zookeeper_dir: /opt
+zookeeper_systemd_service: /etc/systemd/system/zookeeper.service
 # List of dict (i.e. {zookeeper_hosts:[{host:,id:},{host:,id:},...]})
 # zookeeper_hosts:
 #   - host: "{{inventory_hostname}}" # the machine running
diff --git a/ansible/roles/zookeeper-upgrade/handlers/main.yml b/ansible/roles/zookeeper-upgrade/handlers/main.yml
index 3818cc1092..377aac88a9 100644
--- a/ansible/roles/zookeeper-upgrade/handlers/main.yml
+++ b/ansible/roles/zookeeper-upgrade/handlers/main.yml
@@ -1,3 +1,4 @@
 ---
-- name: Restart zookeeper
-  service: name=zookeeper state=restarted
+- name: Restart zookeeper 
+  systemd: name=zookeeper state=restarted
+  become: yes
diff --git a/ansible/roles/zookeeper-upgrade/tasks/Debian.yml b/ansible/roles/zookeeper-upgrade/tasks/Debian.yml
index c0b7efca7f..4d7bffc92e 100644
--- a/ansible/roles/zookeeper-upgrade/tasks/Debian.yml
+++ b/ansible/roles/zookeeper-upgrade/tasks/Debian.yml
@@ -1,41 +1,68 @@
----
-- name: Update apt cache
-  apt: update_cache=yes cache_valid_time={{apt_cache_timeout}}
+- name: download and unarchive zookeeper binary
+  become: yes
+  #  become_user: "{{ zookeeper_user }}"
+  unarchive: 
+    src: "{{ zookeeper_url }}"
+    dest: "{{ zookeeper_dir }}"
+    mode: 0755
+    group: "{{ zookeeper_user }}"
+    owner: "{{ zookeeper_user }}"
+    remote_src: True
 
-- name: Apt install required system packages.
-  apt: pkg={{item}} state=present
-  with_items:
-    - zookeeper
-    - zookeeperd
+- name: check if the zookeeper dir already exists
+  stat:
+    path: "{{ zookeeper_dir }}/zookeeper"
+  register: folder_exists
+
+- name: rename the dir name
+  become: yes
+  shell: mv "apache-zookeeper-{{ zookeeper_version }}-bin" zookeeper
+  #group: "{{ zookeeper_user }}"
+  #owner: "{{ zookeeper_user }}"
+  args:
+    chdir: "{{ zookeeper_dir }}"
+  when: folder_exists.stat.exists == false
 
 - name: Registering server_id
   set_fact:
-    server_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+    server_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
 
 - name: Overwrite myid file.
-  template: src=myid.j2 dest=/etc/zookeeper/conf/myid
+  template: src=myid.j2 dest={{ zookeeper_dir }}/zookeeper/myid
   notify:
     - Restart zookeeper
 
 - name: Overwrite log4j file.
-  template: src=log4j.properties.j2 dest=/etc/zookeeper/conf/log4j.properties
+  template: src=log4j.properties.j2 dest={{ zookeeper_dir }}/zookeeper/conf/log4j.properties
   notify:
     - Restart zookeeper
 
 - name: Overwrite env file.
-  template: src=environment.j2 dest=/etc/zookeeper/conf/environment
+  template: src=environment.j2 dest={{ zookeeper_dir }}/zookeeper/conf/environment
   notify:
     - Restart zookeeper
 
 - name: Overwrite default config file
-  template: src=zoo.cfg.j2 dest=/etc/zookeeper/conf/zoo.cfg
+  template: src=zoo.cfg.j2 dest={{ zookeeper_dir }}/zookeeper/conf/zoo.cfg
   notify:
     - Restart zookeeper
 
-- name: Start zookeeper service
-  service: name=zookeeper state=started enabled=yes
+- name: Change ownership of zookeeper installation
+  become: yes
+  file: path={{ zookeeper_dir }}/zookeeper owner={{ zookeeper_user }} group={{ zookeeper_user }} state=directory recurse=yes
+
+- name: create systemd config
+  template: dest={{zookeeper_systemd_service}} owner=root group=root mode=644 src=zookeeper.service.j2
+  notify:
+    - Restart zookeeper
+
+- name: just force systemd to reread configs
+  systemd: daemon_reload=yes
+
+- name: Enable and Start zookeeper
+  systemd: name=zookeeper state=started enabled=yes
+  become: yes
 
 - name: wait for zookeeper port
   wait_for: host={{zookeeper_listen_address| default('localhost')}} port={{zookeeper_port}} state=started timeout=30
   when: verify
-
diff --git a/ansible/roles/zookeeper-upgrade/tasks/main.yml b/ansible/roles/zookeeper-upgrade/tasks/main.yml
index 3e8a8c09e3..2b1cf6892a 100644
--- a/ansible/roles/zookeeper-upgrade/tasks/main.yml
+++ b/ansible/roles/zookeeper-upgrade/tasks/main.yml
@@ -1,4 +1,11 @@
 ---
+
+- name: setup group
+  group: name={{ zookeeper_user }} system=yes
+
+- name: Setup user
+  user: name={{ zookeeper_user }} system=yes group={{ zookeeper_user }}
+
 - include: Debian.yml
   when: ansible_os_family == 'Debian'
 
diff --git a/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2 b/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2
index 10eb5c0e78..dbdca6a728 100644
--- a/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2
+++ b/ansible/roles/zookeeper-upgrade/templates/zoo.cfg.j2
@@ -1,12 +1,12 @@
 tickTime={{ tick_time }}
-dataDir={{ data_dir }}
-dataLogDir={{ log_dir }}
+dataDir={{ zookeeper_data_dir }}
+dataLogDir={{ zookeeper_data_dir }}
 clientPort={{ client_port }}
 initLimit={{ init_limit }}
 syncLimit={{ sync_limit }}
 maxClientCnxns={{ maxclinetconnection_limit }}
-
-
+{% if zookeeper_group | length > 1 %}
 {% for host in zookeeper_group %}
 server.{{hostvars[host]['server_id']}}={{ host }}:2888:3888
 {% endfor %}
+{% endif %}
diff --git a/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2 b/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2
new file mode 100644
index 0000000000..c33698d2b6
--- /dev/null
+++ b/ansible/roles/zookeeper-upgrade/templates/zookeeper.service.j2
@@ -0,0 +1,16 @@
+[Unit]
+Description=ZooKeeper Service
+Documentation=http://zookeeper.apache.org
+Requires=network.target
+After=network.target
+
+[Service]
+Type=forking
+User=zookeeper
+Group=zookeeper
+ExecStart={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh start {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg
+ExecStop={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh stop {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg
+ExecReload={{ zookeeper_dir }}/zookeeper/bin/zkServer.sh restart {{ zookeeper_dir }}/zookeeper/conf/zoo.cfg
+
+[Install]
+WantedBy=default.target
diff --git a/ansible/samza_deploy.yml b/ansible/samza_deploy.yml
deleted file mode 100644
index 9ac34de840..0000000000
--- a/ansible/samza_deploy.yml
+++ /dev/null
@@ -1,48 +0,0 @@
----
-- name: Move the selected samza app tar files to another dir
-  hosts: localhost
-  tasks:
-    - name: find the selected samza app tar files path
-      find:
-        paths: "{{job_workspace}}/{{ samza_tar_files_localpath }}/allfiles"
-        patterns: "{{ job_names['%s'|format(item)].job_file_name }}*"
-        recurse: yes
-      with_items: "{{ job_names_to_kill.split(',') }}"
-      register: existing_files
-
-    - name: Copy the selected samza app tar files to jobs folder
-      copy:
-       src: "{{ item }}"
-       dest: "{{job_workspace}}/{{ samza_tar_files_localpath }}/jobs"
-      with_items:
-         - "{{ existing_files | json_query('results[].files[].path') }}"
-
-- name: "Start Nodemanager on Slaves if stopped"
-  hosts: "yarn-slave"
-  vars:
-    hadoop_version: 2.7.2
-  become: yes
-  pre_tasks:
-    - name: Ensure yarn nodemanager is running
-      become_user: hduser
-      shell: |
-         (ps aux | grep yarn-hduser-nodemanager | grep -v grep) \
-         || /usr/local/hadoop/sbin/yarn-daemon.sh --config /usr/local/hadoop-{{ hadoop_version }}/conf/ start nodemanager \
-         || sleep 10
-
-    - name: install imagemagick
-      apt: name=imagemagick state=present update_cache=yes
-
-- name: "Deploy Samza jobs"
-  hosts: "yarn-master"
-  become: yes
-  become_user: hduser
-  vars_files:
-    - "{{ inventory_dir }}/secrets.yml"
-  pre_tasks:
-    - name: Ensure yarn resource manager is running
-      shell: |
-        (ps aux | grep yarn-hduser-resourcemanager | grep -v grep) \
-        || /usr/local/hadoop/sbin/yarn-daemon.sh --config /usr/local/hadoop-{{ hadoop_version }}/conf/ start resourcemanager
-  roles:
-    - samza-jobs
diff --git a/ansible/spark-cluster-job-submit.yml b/ansible/spark-cluster-job-submit.yml
new file mode 100644
index 0000000000..ba4e017a23
--- /dev/null
+++ b/ansible/spark-cluster-job-submit.yml
@@ -0,0 +1,11 @@
+---
+- hosts: local
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    AZURE_STORAGE_ACCOUNT: "{{sunbird_private_storage_account_name}}"
+    AZURE_STORAGE_KEY: "{{sunbird_private_storage_account_key}}"
+  roles:
+    - data-products-deploy
+
diff --git a/data-pipeline-flink/assessment-aggregator/pom.xml b/data-pipeline-flink/assessment-aggregator/pom.xml
index 39f46e176e..6b7a921a70 100644
--- a/data-pipeline-flink/assessment-aggregator/pom.xml
+++ b/data-pipeline-flink/assessment-aggregator/pom.xml
@@ -27,7 +27,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -46,16 +46,22 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>mockwebserver</artifactId>
+            <version>4.4.0</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>it.ozimov</groupId>
             <artifactId>embedded-redis</artifactId>
@@ -65,7 +71,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -96,7 +102,7 @@
         <dependency>
             <groupId>org.cassandraunit</groupId>
             <artifactId>cassandra-unit</artifactId>
-            <version>3.1.1.0</version>
+            <version>3.11.2.0</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -105,6 +111,14 @@
     		<sourceDirectory>src/main/scala</sourceDirectory>
 		    <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -153,10 +167,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf b/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf
index 1cd170e190..dcc342732a 100644
--- a/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf
+++ b/data-pipeline-flink/assessment-aggregator/src/main/resources/assessment-aggregator.conf
@@ -4,16 +4,33 @@ kafka {
   input.topic = ${job.env}".telemetry.assess"
   failed.topic= ${job.env}".telemetry.assess.failed"
   groupId = ${job.env}"-assessment-aggregator-group"
+  output.certissue.topic =  ${job.env}".issue.certificate.request"
 }
 
 task {
+  consumer.parallelism = 1
+  downstream.parallelism = 1
   assessaggregator {
     parallelism = 1
   }
+  scoreaggregator {
+    parallelism = 1
+  }
 }
 
 lms-cassandra {
   keyspace = "sunbird_courses"
   table = "assessment_aggregator"
   questionudttype= "question"
+  enrolmentstable = "user_enrolments"
+  activitytable = "user_activity_agg"
+}
+redis {
+  database {
+    relationCache.id = 10
+    contentCache.id = 5
+  }
 }
+assessment.skip.missingRecords = false
+content.read.api = "http://dev.sunbirded.org/api/content/v1/read/"
+user.activity.agg.type="attempt_metrics"
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties b/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties
deleted file mode 100644
index dda4b21056..0000000000
--- a/data-pipeline-flink/assessment-aggregator/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=telemetry-extractor.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala
index 60623bc77e..d77a325926 100644
--- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/domain/Event.scala
@@ -1,8 +1,6 @@
 package org.sunbird.dp.assessment.domain
 
 import java.util
-
-import com.google.gson.internal.LinkedTreeMap
 import org.sunbird.dp.core.domain.{Events, EventsPath}
 
 class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
@@ -10,42 +8,40 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
   private val jobName = "AssessmentAggregator"
 
   def assessmentEts: Long = {
-    val ets = telemetry.read[Double]("assessmentTs").get
-    ets.asInstanceOf[Double].longValue
+    val ets = telemetry.read[Long]("assessmentTs").get
+    ets.longValue()
 
   }
 
-
   def courseId: String = {
-   telemetry.read("courseId").get
+   telemetry.read[String]("courseId").get
   }
 
   def contentId: String = {
-    telemetry.read("contentId").get
+    telemetry.read[String]("contentId").get
   }
 
   def batchId: String = {
-    telemetry.read("batchId").get
+    telemetry.read[String]("batchId").get
   }
 
   def attemptId: String = {
-    telemetry.read("attemptId").get
+    telemetry.read[String]("attemptId").get
   }
 
   def userId: String = {
-    telemetry.read("userId").get
+    telemetry.read[String]("userId").get
   }
 
-  def assessEvents: util.List[LinkedTreeMap[String, AnyRef]] = {
-    telemetry.read("events").get
+  def assessEvents: util.ArrayList[util.Map[String, AnyRef]] = {
+    telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("events").getOrElse(null)
   }
 
   def markFailed(errorMsg: String): Unit = {
     telemetry.addFieldIfAbsent(EventsPath.FLAGS_PATH, new util.HashMap[String, Boolean])
-      telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef])
+    telemetry.addFieldIfAbsent("metadata", new util.HashMap[String, AnyRef])
     telemetry.add("metadata.validation_error", errorMsg)
     telemetry.add("metadata.src", jobName)
-
   }
 
 }
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala
index e027e632be..0bbfa3ec1b 100644
--- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/AssessmentAggregatorFunction.scala
@@ -5,11 +5,10 @@ import java.math.BigDecimal
 import java.sql.Timestamp
 import java.text.DecimalFormat
 import java.util
-
+import java.util.UUID
 import com.datastax.driver.core.querybuilder.QueryBuilder
 import com.datastax.driver.core.{Row, UDTValue, UserType}
 import com.google.gson.Gson
-import com.google.gson.internal.LinkedTreeMap
 import com.google.gson.reflect.TypeToken
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.configuration.Configuration
@@ -18,146 +17,289 @@ import org.joda.time.{DateTime, DateTimeZone}
 import org.slf4j.LoggerFactory
 import org.sunbird.dp.assessment.domain.Event
 import org.sunbird.dp.assessment.task.AssessmentAggregatorConfig
+import org.sunbird.dp.contentupdater.core.util.RestUtil
+import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
 import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
-import org.sunbird.dp.core.util.CassandraUtil
+import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil}
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable
 
-case class Question(id: String, maxscore: Double, params: util.List[LinkedTreeMap[String, Any]], title: String, `type`: String, desc: String)
+case class Question(id: String, maxscore: Double, params: util.List[util.HashMap[String, Any]], title: String, `type`: String, desc: String)
 
-case class QuestionData(resvalues: util.List[LinkedTreeMap[String, Any]], duration: Double, score: Double, item: Question)
+case class QuestionData(resvalues: util.List[util.HashMap[String, Any]], duration: Double, score: Double, item: Question)
 
-case class AssessEvent(ets: Double, edata: QuestionData)
+case class AssessEvent(ets: Long, edata: QuestionData)
 
 
 case class Aggregate(totalScore: Double, totalMaxScore: Double, grandTotal: String, questionsList: List[UDTValue])
 
-
 class AssessmentAggregatorFunction(config: AssessmentAggregatorConfig,
                                    @transient var cassandraUtil: CassandraUtil = null
                                   )(implicit val mapTypeInfo: TypeInformation[Event])
   extends BaseProcessFunction[Event, Event](config) {
 
-    val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType
-    private[this] val logger = LoggerFactory.getLogger(classOf[AssessmentAggregatorFunction])
-    var questionType: UserType = _
-    private val df = new DecimalFormat("0.0#")
-
-    override def metricsList() = List(config.dbUpdateCount, config.dbReadCount, config.failedEventCount, config.batchSuccessCount, config.skippedEventCount)
-
-
-    override def open(parameters: Configuration): Unit = {
-        super.open(parameters)
-        cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort)
-        questionType = cassandraUtil.getUDTType(config.dbKeyspace, config.dbudtType)
-    }
-
-    override def close(): Unit = {
-        super.close()
+  val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType
+  private[this] val logger = LoggerFactory.getLogger(classOf[AssessmentAggregatorFunction])
+  private var dataCache: DataCache = _
+  private var contentCache: DataCache = _
+  var questionType: UserType = _
+  private var restUtil: RestUtil = _
+
+
+  override def metricsList() = List(config.dbUpdateCount, config.dbReadCount,
+    config.failedEventCount, config.batchSuccessCount,
+    config.skippedEventCount, config.cacheHitCount, config.cacheHitMissCount, config.certIssueEventsCount, config.apiHitFailedCount, config.apiHitSuccessCount, config.ignoredEventsCount, config.recomputeAggEventCount)
+
+
+  override def open(parameters: Configuration): Unit = {
+    super.open(parameters)
+    dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.relationCacheNode, List())
+    dataCache.init()
+    contentCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.contentCacheNode, List())
+    contentCache.init()
+    cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort)
+    questionType = cassandraUtil.getUDTType(config.dbKeyspace, config.dbudtType)
+    restUtil = new RestUtil()
+  }
+
+  override def close(): Unit = {
+    super.close()
+  }
+
+  def getListValues(values: util.List[util.HashMap[String, Any]]): mutable.Buffer[util.Map[String, Any]] = {
+    values.asScala.map { res =>
+      res.asScala.map {
+        case (key, value) => key -> (if (null != value && !value.isInstanceOf[String]) new Gson().toJson(value) else value)
+      }.toMap.asJava
     }
-
-    def getListValues(values: util.List[LinkedTreeMap[String, Any]]): mutable.Buffer[util.Map[String, Any]] = {
-        values.asScala.map { res =>
-            res.asScala.map {
-                case (key, value) => key -> (if (null != value && !value.isInstanceOf[String]) new Gson().toJson(value) else value)
-            }.toMap.asJava
-        }
-    }
-
-    /**
-      * Method to write the assess event to cassandra table
-      *
-      * @param event   - Assess Batch Events
-      * @param context - Process Context
-      */
-    override def processElement(event: Event,
-                                context: ProcessFunction[Event, Event]#Context,
-                                metrics: Metrics): Unit = {
-        try {
-            var totalScore = 0.0
-            var totalMaxScore = 0.0
-            val assessment = getAssessment(event)
-            val assessEvents = event.assessEvents.asScala
-
-            val sortAndFilteredEvents = assessEvents.map(event => {
-                AssessEvent(event.get("ets").asInstanceOf[Double], new Gson().fromJson(new Gson().toJson(event.get("edata")),
-                    classOf[QuestionData]))
-            }).sortWith(_.ets > _.ets).groupBy(_.edata.item.id).map(_._2.head)
-
-            val result = sortAndFilteredEvents.map(event => {
-                totalScore = totalScore + event.edata.score
-                totalMaxScore = totalMaxScore + event.edata.item.maxscore
-                getQuestion(event.edata, event.ets.longValue())
-            })
-
-            val grandTotal = String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore))
-
-            if (null == assessment) {
-                saveAssessment(event, Aggregate(totalScore, totalMaxScore, grandTotal, result.toList), new DateTime().getMillis)
-                metrics.incCounter(config.dbUpdateCount)
-                metrics.incCounter(config.batchSuccessCount)
-            }
-            else {
-                metrics.incCounter(config.dbReadCount)
-                if (event.assessmentEts > assessment.getTimestamp("last_attempted_on").getTime) {
-                    saveAssessment(event, Aggregate(totalScore, totalMaxScore, grandTotal, result.toList),
-                        assessment.getTimestamp("created_on").getTime)
-                    metrics.incCounter(config.dbUpdateCount)
-                    metrics.incCounter(config.batchSuccessCount)
-                }
-                else {
-                    metrics.incCounter(config.skippedEventCount)
-                }
-            }
-        } catch {
-            case ex: Exception =>
-                logger.info("Assessment Failed with exception :", ex)
-                event.markFailed(ex.getMessage)
+  }
+
+  /**
+   * Method to write the assess event to cassandra table
+   *
+   * @param event   - Assess Batch Events
+   * @param context - Process Context
+   */
+  override def processElement(event: Event,
+                              context: ProcessFunction[Event, Event]#Context,
+                              metrics: Metrics): Unit = {
+    try {
+      // Validating the contentId
+      if (isValidContent(event.courseId, event.contentId)(metrics)) {
+        val assessEvents = event.assessEvents.asScala
+        if(null != assessEvents && !assessEvents.isEmpty) {
+          val sortAndFilteredEvents: List[AssessEvent] = getUniqueQuestions(assessEvents = assessEvents.toList)
+          if (config.skipMissingRecords) { // Skip configuration to enable the force filter of duplicate questions based on the question meta
+            val totalQuestions = getTotalQuestionsCount(event.contentId)(metrics)
+            logger.info(s"The total Question count value - ContentId:${event.contentId}, TotalQuestionCount:$totalQuestions")
+            if (totalQuestions == null) {
+              updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context)
+            } else {
+              /**
+               * If the totalQuestions from the content Meta and Events count are matching
+               * Then we are computing the score metrics and updating the table
+               */
+              if (sortAndFilteredEvents.size <= totalQuestions.asInstanceOf[Int]) {
+                updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context)
+              } else {
+                /**
+                 * if the totalQuestions Count value is not matching with the event size then job should skip those events and increase the metrics value
+                 */
                 context.output(config.failedEventsOutputTag, event)
-                metrics.incCounter(config.failedEventCount)
+                metrics.incCounter(config.ignoredEventsCount)
+              }
+            }
+          } else {
+            /**
+             * If the config.skipMissingRecords = false then it will work as previous logic, no further validation
+             */
+            updateDB(scoreMetrics = computeScoreMetrics(sortAndFilteredEvents), event = event)(metrics, context)
+          }
+        } else {
+          recomputeAggregates(event)(metrics, context)
         }
+        
+      } else {
+        context.output(config.failedEventsOutputTag, event)
+        metrics.incCounter(config.failedEventCount)
+      }
+    } catch {
+      case ex: Exception =>
+        ex.printStackTrace()
+        logger.info("Assessment Failed with exception :", ex.getMessage)
+        event.markFailed(ex.getMessage)
+        context.output(config.failedEventsOutputTag, event)
+        metrics.incCounter(config.failedEventCount)
     }
-
-    def getAssessment(event: Event): Row = {
-
-        val query = QueryBuilder.select("last_attempted_on", "created_on")
-          .from(config.dbKeyspace, config.dbTable).
-          where(QueryBuilder.eq("attempt_id", event.attemptId))
-          .and(QueryBuilder.eq("batch_id", event.batchId))
-          .and(QueryBuilder.eq("user_id", event.userId))
-          .and(QueryBuilder.eq("content_id", event.contentId))
-          .and(QueryBuilder.eq("course_id", event.courseId)).toString
-
-        cassandraUtil.findOne(query)
-
+  }
+
+  def updateDB(scoreMetrics: Aggregate, event: Event)(metrics: Metrics, context: ProcessFunction[Event, Event]#Context): Unit = {
+    val assessmentDBResult = getAssessment(event)
+    if (null == assessmentDBResult) {
+      saveAssessment(event, Aggregate(scoreMetrics.totalScore, scoreMetrics.totalMaxScore, scoreMetrics.grandTotal, scoreMetrics.questionsList), new DateTime().getMillis)
+      metrics.incCounter(config.dbUpdateCount)
+      metrics.incCounter(config.batchSuccessCount)
+      context.output(config.scoreAggregateTag, event)
+      createIssueCertEvent(event, context, metrics)
     }
-
-
-    def saveAssessment(batchEvent: Event, aggregate: Aggregate, createdOn: Long): Unit = {
-        val query = QueryBuilder.insertInto(config.dbKeyspace, config.dbTable)
-          .value("course_id", batchEvent.courseId).value("batch_id", batchEvent.batchId).value("user_id", batchEvent.userId)
-          .value("content_id", batchEvent.contentId).value("attempt_id", batchEvent.attemptId)
-          .value("updated_on", new DateTime(DateTimeZone.UTC).getMillis).value("created_on", createdOn)
-          .value("last_attempted_on", batchEvent.assessmentEts).value("total_score", aggregate.totalScore)
-          .value("total_max_score", aggregate.totalMaxScore)
-          .value("question", aggregate.questionsList.asJava).value("grand_total", aggregate.grandTotal).toString
-
-        cassandraUtil.upsert(query)
-        logger.info("Successfully Aggregated the batch event - batchid: "
-          + batchEvent.batchId + " ,userid: " + batchEvent.userId + " ,couserid: "
-          + batchEvent.courseId + " ,contentid: " + batchEvent.contentId, "attempid" + batchEvent.attemptId)
+    else {
+      metrics.incCounter(config.dbReadCount)
+      if (event.assessmentEts > assessmentDBResult.getTimestamp("last_attempted_on").getTime) {
+        saveAssessment(event, Aggregate(scoreMetrics.totalScore, scoreMetrics.totalMaxScore, scoreMetrics.grandTotal, scoreMetrics.questionsList),
+          assessmentDBResult.getTimestamp("created_on").getTime)
+        metrics.incCounter(config.dbUpdateCount)
+        metrics.incCounter(config.batchSuccessCount)
+        context.output(config.scoreAggregateTag, event)
+        createIssueCertEvent(event, context, metrics)
+      }
+      else {
+        metrics.incCounter(config.skippedEventCount)
+      }
     }
-
-    def getQuestion(questionData: QuestionData, assessTs: Long): UDTValue =
-
-        questionType.newValue().setString("id", questionData.item.id).setDouble("max_score", questionData.item.maxscore)
-          .setDouble("score", questionData.score)
-          .setString("type", questionData.item.`type`)
-          .setString("title", questionData.item.title)
-          .setList("resvalues", getListValues(questionData.resvalues).asJava).setList("params", getListValues(questionData.item.params).asJava)
-          .setString("description", questionData.item.desc)
-          .setDecimal("duration", BigDecimal.valueOf(questionData.duration)).setTimestamp("assess_ts", new Timestamp(assessTs))
+  }
+
+  def getUniqueQuestions(assessEvents: List[util.Map[String, AnyRef]]): List[AssessEvent] = {
+    assessEvents.map(event => {
+      AssessEvent(event.get("ets").asInstanceOf[Long], new Gson().fromJson(new Gson().toJson(event.get("edata")), classOf[QuestionData]))
+    }).sortWith(_.ets > _.ets).groupBy(_.edata.item.id).map(_._2.head).toList
+  }
+
+  def computeScoreMetrics(events: List[AssessEvent]): Aggregate = {
+    var totalScore = 0.0
+    var totalMaxScore = 0.0
+    val df = new DecimalFormat("0.0#")
+    val questions = events.map(event => {
+      totalScore = totalScore + event.edata.score
+      totalMaxScore = totalMaxScore + event.edata.item.maxscore
+      getQuestion(event.edata, event.ets.longValue())
+    })
+    val grandTotal = String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore))
+    Aggregate(totalScore = totalScore, totalMaxScore = totalMaxScore, grandTotal = grandTotal, questionsList = questions)
+  }
+
+  def getAssessment(event: Event): Row = {
+
+    val query = QueryBuilder.select("last_attempted_on", "created_on")
+      .from(config.dbKeyspace, config.dbTable).
+      where(QueryBuilder.eq("attempt_id", event.attemptId))
+      .and(QueryBuilder.eq("batch_id", event.batchId))
+      .and(QueryBuilder.eq("user_id", event.userId))
+      .and(QueryBuilder.eq("content_id", event.contentId))
+      .and(QueryBuilder.eq("course_id", event.courseId)).toString
+
+    cassandraUtil.findOne(query)
+
+  }
+  def isValidContent(courseId: String, contentId: String)(metrics: Metrics): Boolean = {
+    val leafNodes = dataCache.getKeyMembers(key = s"$courseId:$courseId:leafnodes")
+    if (!leafNodes.isEmpty) {
+      metrics.incCounter(config.cacheHitCount)
+      leafNodes.contains(contentId)
+    } else {
+      metrics.incCounter(config.cacheHitMissCount)
+      true
+    }
+  }
+
+  def saveAssessment(batchEvent: Event, aggregate: Aggregate, createdOn: Long): Unit = {
+    val query = QueryBuilder.insertInto(config.dbKeyspace, config.dbTable)
+      .value("course_id", batchEvent.courseId).value("batch_id", batchEvent.batchId).value("user_id", batchEvent.userId)
+      .value("content_id", batchEvent.contentId).value("attempt_id", batchEvent.attemptId)
+      .value("updated_on", new DateTime(DateTimeZone.UTC).getMillis).value("created_on", createdOn)
+      .value("last_attempted_on", batchEvent.assessmentEts).value("total_score", aggregate.totalScore)
+      .value("total_max_score", aggregate.totalMaxScore)
+      .value("question", aggregate.questionsList.asJava).value("grand_total", aggregate.grandTotal).toString
+
+    cassandraUtil.upsert(query)
+    logger.info("Successfully Aggregated the batch event - batchid: "
+      + batchEvent.batchId + " ,userid: " + batchEvent.userId + " ,couserid: "
+      + batchEvent.courseId + " ,contentid: " + batchEvent.contentId + "attempid" + batchEvent.attemptId)
+  }
+
+  def getQuestion(questionData: QuestionData, assessTs: Long): UDTValue = {
+
+    questionType.newValue().setString("id", questionData.item.id).setDouble("max_score", questionData.item.maxscore)
+      .setDouble("score", questionData.score)
+      .setString("type", questionData.item.`type`)
+      .setString("title", questionData.item.title)
+      .setList("resvalues", getListValues(questionData.resvalues).asJava).setList("params", getListValues(questionData.item.params).asJava)
+      .setString("description", questionData.item.desc)
+      .setDecimal("duration", BigDecimal.valueOf(questionData.duration)).setTimestamp("assess_ts", new Timestamp(assessTs))
+  }
+
+  /**
+   * Generation of Certificate Issue event for the enrolment completed users to validate and generate certificate.
+   *
+   * @param batchEvent
+   * @param context
+   * @param metrics
+   */
+  def createIssueCertEvent(batchEvent: Event, context: ProcessFunction[Event, Event]#Context,
+                           metrics: Metrics): Unit = {
+    val ets = System.currentTimeMillis
+    val mid = s"""LP.${ets}.${UUID.randomUUID}"""
+    val event =
+      s"""{"eid": "BE_JOB_REQUEST",
+         |"ets": ${ets},
+         |"mid": "${mid}",
+         |"actor": {"id": "Course Certificate Generator","type": "System"},
+         |"context": {"pdata": {"ver": "1.0","id": "org.sunbird.platform"}},
+         |"object": {"id": "${batchEvent.batchId}_${batchEvent.courseId}","type": "CourseCertificateGeneration"},
+         |"edata": {"userIds": ["${batchEvent.userId}"],"action": "issue-certificate","iteration": 1, "trigger": "auto-issue","batchId": "${batchEvent.batchId}","reIssue": false,"courseId": "${batchEvent.courseId}"}}"""
+        .stripMargin.replaceAll("\n", "")
+    context.output(config.certIssueOutputTag, event)
+    metrics.incCounter(config.certIssueEventsCount)
+  }
+
+  def getQuestionCountFromCache(contentId: String)(metrics: Metrics) = {
+    val result = contentCache.getWithRetry(contentId)
+    metrics.incCounter(config.cacheHitCount)
+    result.getOrElse("totalQuestions", 0.0).asInstanceOf[Double].toInt
+  }
+
+  def getQuestionCountFromAPI(contentId: String)(metrics: Metrics) = {
+    val contentReadResp = JSONUtil.deserialize[util.HashMap[String, AnyRef]](restUtil.get(config.contentReadAPI.concat(contentId)))
+    if (contentReadResp.get("responseCode").asInstanceOf[String].toUpperCase.equalsIgnoreCase("OK")) {
+      metrics.incCounter(config.apiHitSuccessCount)
+      val result = contentReadResp.getOrDefault("result", new util.HashMap()).asInstanceOf[util.Map[String, AnyRef]]
+      val content = result.getOrDefault("content", new util.HashMap()).asInstanceOf[util.Map[String, Any]]
+      val totalQuestions = content.getOrDefault("totalQuestions", null)
+      logger.info(s"Fetched the totalQuestion Value from the Content Read API - ContentId:$contentId, TotalQuestionCount:$totalQuestions")
+      totalQuestions
+    } else {
+      metrics.incCounter(config.apiHitFailedCount)
+      logger.info(s"API Failed to Fetch the TotalQuestion Count - ContentId:$contentId, ResponseCode - ${contentReadResp.get("responseCode")} ")
+      throw new Exception(s"Failed to fetch the content meta for the content ID: $contentId") // Job should stop if the api has failed
+    }
+  }
+
+  def getTotalQuestionsCount(contentId: String)(metrics: Metrics) = {
+    val totalQuestionsCountFromCache = getQuestionCountFromCache(contentId)(metrics)
+    logger.info(s" Total Question Count Value From Redis - ContentId:$contentId, TotalQuestionCount - ${totalQuestionsCountFromCache} ")
+    if (isNegligibleValue(totalQuestionsCountFromCache)) {
+      metrics.incCounter(config.cacheHitMissCount)
+      getQuestionCountFromAPI(contentId)(metrics)
+    } else {
+      totalQuestionsCountFromCache
+    }
+  }
+
+  def isNegligibleValue(value: Int): Boolean = {
+    if (value == 0) true else false
+  }
+
+  /**
+   * Pushes event to compute the score aggregates and pushes certificate issue event to kafka
+   * @param event
+   * @param metrics
+   * @param context
+   */
+  def recomputeAggregates(event: Event)(metrics: Metrics, context: ProcessFunction[Event, Event]#Context) = {
+    metrics.incCounter(config.recomputeAggEventCount)
+    context.output(config.scoreAggregateTag, event)
+    createIssueCertEvent(event, context, metrics)
+  }
 
 }
 
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala
new file mode 100644
index 0000000000..fabda4fbf8
--- /dev/null
+++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/functions/UserScoreAggregateFunction.scala
@@ -0,0 +1,110 @@
+package org.sunbird.dp.assessment.functions
+
+import java.lang.reflect.Type
+import java.util
+import java.util.Date
+
+import com.datastax.driver.core.Row
+import com.datastax.driver.core.querybuilder.QueryBuilder
+import com.google.gson.Gson
+import com.google.gson.reflect.TypeToken
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.assessment.domain.Event
+import org.apache.flink.configuration.Configuration
+import org.sunbird.dp.assessment.task.AssessmentAggregatorConfig
+import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
+import org.sunbird.dp.core.util.CassandraUtil
+
+import scala.collection.JavaConverters._
+
+case class AggDetails(attempt_id: String, last_attempted_on: Date, score: Double, content_id: String, max_score: Double, `type`: String)
+
+case class UserActivityAgg(aggregates: Map[String, Double], aggDetails: List[String])
+
+class UserScoreAggregateFunction(config: AssessmentAggregatorConfig,
+                                 @transient var cassandraUtil: CassandraUtil = null
+                                )(implicit val mapTypeInfo: TypeInformation[Event])
+  extends BaseProcessFunction[Event, Event](config) {
+
+  val mapType: Type = new TypeToken[util.Map[String, AnyRef]]() {}.getType
+  private[this] val logger = LoggerFactory.getLogger(classOf[UserScoreAggregateFunction])
+
+  override def metricsList() = List(config.dbScoreAggUpdateCount, config.dbScoreAggReadCount,
+    config.failedEventCount, config.batchSuccessCount,
+    config.skippedEventCount)
+
+  override def open(parameters: Configuration): Unit = {
+    super.open(parameters)
+    cassandraUtil = new CassandraUtil(config.dbHost, config.dbPort)
+  }
+
+  override def close(): Unit = {
+    super.close()
+  }
+
+  def getAggregateDetails(assessAggRows: List[Row]): List[String] = {
+    if (null != assessAggRows && !assessAggRows.isEmpty) {
+      assessAggRows.map { row =>
+        val aggMap = AggDetails(row.getString("attempt_id"), row.getTimestamp("last_attempted_on"),
+          row.getDouble("score"), row.getString("content_id"), row.getDouble("total_max_score"), config.aggType)
+        new Gson().toJson(aggMap)
+      }.toList
+    } else List()
+  }
+
+  def getAggregates(assessAggRows: List[Row]): Map[String, Double] = {
+    if (null != assessAggRows && !assessAggRows.isEmpty) {
+      val attemptGroupList = assessAggRows.groupBy(row => row.getString("content_id")).values
+      attemptGroupList.map(row => {
+        val scoreRow = row.maxBy(r => r.getDouble("score"))
+
+        Map(s"score:${scoreRow.getString("content_id")}" -> scoreRow.getDouble("score"),
+          s"max_score:${scoreRow.getString("content_id")}" -> scoreRow.getDouble("total_max_score"),
+          s"attempts_count:${scoreRow.getString("content_id")}" -> row.length.toDouble)
+      }).flatten.toMap
+    } else Map[String, Double]()
+  }
+
+  def getBestScore(event: Event): UserActivityAgg = {
+    val query = QueryBuilder.select().column("content_id").column("attempt_id").column("last_attempted_on").column("total_max_score").column("total_score").as("score").from(config.dbKeyspace, config.dbTable)
+      .where(QueryBuilder.eq("course_id", event.courseId)).and(QueryBuilder.eq("batch_id", event.batchId))
+      .and(QueryBuilder.eq("user_id", event.userId))
+    val rows = cassandraUtil.find(query.toString).asScala.toList
+
+    UserActivityAgg(aggregates = getAggregates(rows), aggDetails = getAggregateDetails(rows))
+  }
+
+  def updateUserActivity(event: Event, score: UserActivityAgg): Unit = {
+    val scoreLastUpdatedTime: Map[String, Long] = score.aggregates.map(m => m._1 -> System.currentTimeMillis())
+    val updateQuery = QueryBuilder.update(config.dbKeyspace, config.activityTable)
+      .`with`(QueryBuilder.putAll(config.aggregates, score.aggregates.asJava))
+      .and(QueryBuilder.set(config.aggDetails, score.aggDetails.asJava))
+      .and(QueryBuilder.putAll(config.aggLastUpdated, scoreLastUpdatedTime.asJava))
+      .where(QueryBuilder.eq(config.activityId, event.courseId))
+      .and(QueryBuilder.eq(config.activityType, "Course"))
+      .and(QueryBuilder.eq(config.contextId, "cb:" + event.batchId))
+      .and(QueryBuilder.eq(config.activityUser, event.userId))
+    cassandraUtil.upsert(updateQuery.toString)
+    logger.info("Successfully updated scores in user activity  - batchid: "
+      + event.batchId + " ,userid: " + event.userId + " ,couserid: "
+      + event.courseId)
+  }
+
+  override def processElement(event: Event,
+                              context: ProcessFunction[Event, Event]#Context,
+                              metrics: Metrics): Unit = {
+    val score: UserActivityAgg = getBestScore(event)
+    metrics.incCounter(config.dbScoreAggReadCount)
+    if (score.aggregates.nonEmpty || score.aggDetails.nonEmpty) {
+      updateUserActivity(event, score)
+      metrics.incCounter(config.dbScoreAggUpdateCount)
+    } else {
+      logger.info("No scores to update for batchid: "
+        + event.batchId + " ,userid: " + event.userId + " ,couserid: "
+        + event.courseId)
+    }
+  }
+
+}
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala
index 1a37bfcb06..2fc795681a 100644
--- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorConfig.scala
@@ -9,44 +9,83 @@ import org.sunbird.dp.core.job.BaseJobConfig
 
 class AssessmentAggregatorConfig(override val config: Config) extends BaseJobConfig(config, jobName = "AssessmentAggregatorJob") {
 
-    private val serialVersionUID = 2905979434303791379L
-
-    implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
-
-
-    // Kafka Topics Configurationval kafkaInputTopic: String = config.getString("kafka.input.topic")
-
-    val assessAggregatorParallelism: Int = config.getInt("task.assessaggregator.parallelism")
-    val kafkaInputTopic: String = config.getString("kafka.input.topic")
-    val kafkaFailedTopic: String = config.getString("kafka.failed.topic")
-
-    // Metric List
-    val dbUpdateCount = "db-update-count"
-    val dbReadCount = "db-read-count"
-    val batchSuccessCount = "batch-success-event-count"
-    val failedEventCount = "failed-event-count"
-    val skippedEventCount = "skipped-event-count"
-
-
-    //Cassandra
-
-    val dbTable: String = config.getString("lms-cassandra.table")
-    val dbKeyspace: String = config.getString("lms-cassandra.keyspace")
-    val dbHost: String = config.getString("lms-cassandra.host")
-    val dbPort: Int = config.getInt("lms-cassandra.port")
-    val dbudtType: String = config.getString("lms-cassandra.questionudttype")
-
-    val FAILED_EVENTS_OUTPUT_TAG = "failed-events"
-
-    val failedEventsOutputTag: OutputTag[Event] = OutputTag[Event]("assess-failed-events")
-
-    // Consumers
-    val assessmentAggConsumer = "assessment-agg-consumer"
-
-    // Functions
-    val assessmentAggregatorFunction = "AssessmentAggregatorFunction"
-
-    // Producers
-    val assessFailedEventsSink = "assess-failed-events-sink"
+  private val serialVersionUID = 2905979434303791379L
+
+  implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
+
+
+  // Kafka Topics Configurationval kafkaInputTopic: String = config.getString("kafka.input.topic")
+  // Parallelism configs
+  val assessAggregatorParallelism: Int = config.getInt("task.assessaggregator.parallelism")
+  val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.parallelism")
+  val scoreAggregatorParallelism: Int = config.getInt("task.scoreaggregator.parallelism")
+  override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
+
+  val kafkaInputTopic: String = config.getString("kafka.input.topic")
+  val kafkaFailedTopic: String = config.getString("kafka.failed.topic")
+  val kafkaCertIssueTopic: String = config.getString("kafka.output.certissue.topic")
+
+  // Metric List
+  val dbUpdateCount = "db-update-count"
+  val dbReadCount = "db-read-count"
+  val batchSuccessCount = "batch-success-event-count"
+  val failedEventCount = "failed-event-count"
+  val ignoredEventsCount = "ignored-event-count"
+  val skippedEventCount = "skipped-event-count"
+  val cacheHitCount = "cache-hit-count"
+  val cacheHitMissCount = "cache-hit-miss-count"
+  val certIssueEventsCount = "cert-issue-events-count"
+  val dbScoreAggUpdateCount = "db-score-update-count"
+  val dbScoreAggReadCount = "db-score-read-count"
+  val apiHitSuccessCount = "api-hit-success-count"
+  val apiHitFailedCount = "api-hit-failed-count"
+  val recomputeAggEventCount = "recompute-agg-event-count"
+
+
+  //Cassandra
+
+  val dbTable: String = config.getString("lms-cassandra.table")
+  val dbKeyspace: String = config.getString("lms-cassandra.keyspace")
+  val dbHost: String = config.getString("lms-cassandra.host")
+  val dbPort: Int = config.getInt("lms-cassandra.port")
+  val dbudtType: String = config.getString("lms-cassandra.questionudttype")
+  val enrolmentTable: String = config.getString("lms-cassandra.enrolmentstable")
+  val activityTable: String = config.getString("lms-cassandra.activitytable")
+
+  val FAILED_EVENTS_OUTPUT_TAG = "failed-events"
+
+  val failedEventsOutputTag: OutputTag[Event] = OutputTag[Event]("assess-failed-events")
+  val certIssueOutputTagName = "certificate-issue-events"
+  val certIssueOutputTag: OutputTag[String] = OutputTag[String](certIssueOutputTagName)
+
+  // Consumers
+  val assessmentAggConsumer = "assessment-agg-consumer"
+
+  // Functions
+  val assessmentAggregatorFunction = "AssessmentAggregatorFunction"
+
+  // Producers
+  val assessFailedEventsSink = "assess-failed-events-sink"
+  val certIssueEventSink = "certificate-issue-event-sink"
+  val userScoreAggregateFn = "user-score-aggregator"
+
+  // Cache
+  val relationCacheNode:Int = config.getInt("redis.database.relationCache.id")
+  val contentCacheNode:Int = config.getInt("redis.database.contentCache.id")
+  
+  //UserActivityAgg
+  val scoreAggregateTag: OutputTag[Event] = OutputTag[Event]("score-aggregate-events")
+  val activityType = "activity_type"
+  val activityId = "activity_id"
+  val contextId = "context_id"
+  val activityUser = "user_id"
+  val aggLastUpdated = "agg_last_updated"
+  val aggDetails = "agg_details"
+  val aggregates = "aggregates"
+
+  val aggType = config.getString("user.activity.agg.type")
+
+  val skipMissingRecords: Boolean = config.getBoolean("assessment.skip.missingRecords")
+  val contentReadAPI: String = config.getString("content.read.api")
 
 }
diff --git a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala
index a3dcbb953f..bb6a3399aa 100644
--- a/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/main/scala/org/sunbird/dp/assessment/task/AssessmentAggregatorStreamTask.scala
@@ -6,10 +6,9 @@ import com.typesafe.config.ConfigFactory
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.sunbird.dp.assessment.domain.Event
-import org.sunbird.dp.assessment.functions.AssessmentAggregatorFunction
+import org.sunbird.dp.assessment.functions.{AssessmentAggregatorFunction, UserScoreAggregateFunction}
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.core.util.FlinkUtil
 
@@ -48,15 +47,21 @@ class AssessmentAggregatorStreamTask(config: AssessmentAggregatorConfig, kafkaCo
         implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
         val source = kafkaConnector.kafkaEventSource[Event](config.kafkaInputTopic)
 
-        val aggregatorStream: SingleOutputStreamOperator[Event] =
-            env.addSource(source, config.assessmentAggConsumer).uid(config.assessmentAggConsumer)
-              .rebalance()
+        val aggregatorStream =
+            env.addSource(source, config.assessmentAggConsumer)
+              .uid(config.assessmentAggConsumer).setParallelism(config.kafkaConsumerParallelism).rebalance()
               .process(new AssessmentAggregatorFunction(config))
               .name(config.assessmentAggregatorFunction).uid(config.assessmentAggregatorFunction)
               .setParallelism(config.assessAggregatorParallelism)
 
         aggregatorStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaFailedTopic))
           .name(config.assessFailedEventsSink).uid(config.assessFailedEventsSink)
+          .setParallelism(config.downstreamOperatorsParallelism)
+        aggregatorStream.getSideOutput(config.certIssueOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaCertIssueTopic))
+          .name(config.certIssueEventSink).uid(config.certIssueEventSink)
+          .setParallelism(config.downstreamOperatorsParallelism)
+        aggregatorStream.getSideOutput(config.scoreAggregateTag).process(new UserScoreAggregateFunction(config))
+          .name(config.userScoreAggregateFn).uid(config.userScoreAggregateFn).setParallelism(config.scoreAggregatorParallelism)
         env.execute(config.jobName)
     }
 }
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf
new file mode 100644
index 0000000000..5f8ac8138f
--- /dev/null
+++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/forcevalidate.conf
@@ -0,0 +1,40 @@
+include "base-test.conf"
+
+kafka {
+  input.topic = "flink.telemetry.assess"
+  groupId = "flink-assessment-aggregator-group"
+  failed.topic= "telemetry.assess.failed"
+  output.certissue.topic = "issue.certificate.request"
+}
+
+task {
+  parallelism = 1
+  checkpointing.interval = 60000
+  assessaggregator {
+    parallelism = 1
+  }
+  scoreaggregator {
+      parallelism = 1
+    }
+  consumer.parallelism = 1
+  downstream.parallelism = 1
+}
+
+lms-cassandra {
+  keyspace = "sunbird_courses"
+  table = "assessment_aggregator"
+  questionudttype= "question"
+  enrolmentstable = "user_enrolments"
+  activitytable = "user_activity_agg"
+}
+
+redis {
+  database {
+    relationCache.id = 10
+    contentCache.id = 5
+  }
+}
+
+assessment.skip.missingRecords = true
+content.read.api = "http://127.0.0.1:3000/api/content/v1/read/"
+user.activity.agg.type="attempt_metrics"
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
index 243d024576..fbe8e16690 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
+++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.conf
@@ -4,6 +4,7 @@ kafka {
   input.topic = "flink.telemetry.assess"
   groupId = "flink-assessment-aggregator-group"
   failed.topic= "telemetry.assess.failed"
+  output.certissue.topic = "issue.certificate.request"
 }
 
 task {
@@ -12,10 +13,27 @@ task {
   assessaggregator {
     parallelism = 1
   }
+  scoreaggregator {
+      parallelism = 1
+    }
+  consumer.parallelism = 1
+  downstream.parallelism = 1
 }
 
 lms-cassandra {
   keyspace = "sunbird_courses"
   table = "assessment_aggregator"
   questionudttype= "question"
-}
\ No newline at end of file
+  enrolmentstable = "user_enrolments"
+  activitytable = "user_activity_agg"
+}
+
+redis {
+  database {
+    relationCache.id = 10
+    contentCache.id = 5
+  }
+}
+assessment.skip.missingRecords = false
+content.read.api = "http://127.0.0.1:3000/api/content/v1/read/"
+user.activity.agg.type="attempt_metrics"
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql
index 99b0f0a92f..a401643c44 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql
+++ b/data-pipeline-flink/assessment-aggregator/src/test/resources/test.cql
@@ -1,4 +1,4 @@
-CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1};
+CREATE KEYSPACE IF NOT EXISTS sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1};
 
 
 CREATE TYPE IF NOT EXISTS sunbird_courses.question (
@@ -16,11 +16,11 @@ duration decimal
 
 
 CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator (
+user_id text,
 course_id text,
 batch_id text,
 content_id text,
 attempt_id text,
-user_id text,
 created_on timestamp,
 last_attempted_on timestamp,
 total_max_score double,
@@ -28,10 +28,10 @@ grand_total text,
 question list<frozen<question>>,
 total_score double,
 updated_on timestamp,
-PRIMARY KEY (course_id, batch_id, content_id, attempt_id, user_id)
+PRIMARY KEY ((user_id, course_id), batch_id, content_id, attempt_id)
 );
 
-CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on);
+CREATE INDEX IF NOT EXISTS assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on);
 
 
 /*INSERT INTO sunbird_courses.assessment_aggregator (course_id,batch_id,user_id,content_id,attempt_id,created_on,grand_total
@@ -40,3 +40,37 @@ CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.asse
 '0afe2cc175ffc95b83a4dc8d1af2c47b','2019-11-07 05:38:49.640+0000','2/5','2019-11-07 05:38:05.076+0000',null,
 5,2,'2019-11-07 05:38:49.642+0000');*/
 
+CREATE TABLE IF NOT EXISTS sunbird_courses.user_enrolments (
+userid text,
+courseid text,
+batchid text,
+active boolean,
+addedby text,
+certificates list<frozen<map<text, text>>>,
+completedon timestamp,
+completionpercentage int,
+contentstatus map<text, int>,
+datetime timestamp,
+enrolleddate text,
+issued_certificates list<frozen<map<text, text>>>,
+lastreadcontentid text,
+lastreadcontentstatus int,
+progress int,
+status int,
+PRIMARY KEY (userid, courseid, batchid));
+
+INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f', 'do_2128415652377067521125', '012846671379595264119', 2);
+INSERT INTO sunbird_courses.user_enrolments(userid, courseid, batchid, status) VALUES ('d0d8a341-9637-484c-b871-0c27015af238', 'do_2128410273679114241112', '01284169026368307244', 1);
+
+
+CREATE TABLE IF NOT EXISTS sunbird_courses.user_activity_agg (
+activity_type text,
+activity_id text,
+user_id text,
+context_id text,
+agg map<text, int>,
+agg_last_updated map<text, timestamp>,
+aggregates map<text, double>,
+agg_details list<text>,
+PRIMARY KEY ((activity_type, activity_id, user_id), context_id)
+) WITH CLUSTERING ORDER BY (context_id ASC);
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index cf714de51d..40bba34aee 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -1,16 +1,44 @@
+
 package org.sunbird.dp.fixture
 
 object EventFixture {
 
-    val BATCH_ASSESS_EVENT ="""{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val BATCH_ASSESS_EVENT = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+
+  val SECOND_ATTEMPT_BATCH_ASSESS_EVENT = """{"assessmentTs":1568891729576,"attemptId":"9dd87e24df268ad09a8b0060c0a40262","batchId":"012846671379595264119","contentId":"do_212686723743318016173","courseId":"do_2128415652377067521125","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:cb00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:235815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:7ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:118f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"aqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEE","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521125","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.1","mid":"ASSESS:59d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}],"userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f"}"""
+
+  // CourseId does not have this contentId do_4957349 in the leafnodes list
+  val BATCH_ASSESS_EVENT_WITHOUT_CACHE = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_4957349","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_4957349","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+
+  val LATEST_BATCH_ASSESS_EVENT = """{"assessmentTs":1587988642000,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":1,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362dr776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":1,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+
+  val BATCH_ASSESS__OLDER_EVENT: String = """{"courseId":"do_2128415652377067521125","batchId":"012846671379595264119","contentId":"do_212686723743318016173","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","assessmentTs":1567073236195,"attemptId":"8cd87e24df268ad09a8b0060c0a40271","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}"""
+
+  val BATCH_ASSESS_FAIL_EVENT: String = """{"courseId":"do_312712196780204032110117","batchId":"01271220181664563270","contentId":"505c7c48ac6dc1edc9b08f21db5a571d","userId":"b3541347e18ab916c06ed76aeb0ce57f","assessmentTs":1567073236195,"attemptId":"attempt1","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":"1567073236195","ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}"""
+
+  val BATCH_DUPLICATE_QUESTION_EVENT: String = """{"assessmentTs":1569304757079,"batchId":"01284169026368307244","courseId":"do_2128410273679114241112","userId":"d0d8a341-9637-484c-b871-0c27015af238","attemptId":"90e1a0d12542806389a1a52aaf1fc622","contentId":"do_2128373396098744321673","events":[{"eid":"ASSESS","ets":1569304758743,"ver":"3.1","mid":"ASSESS:5b2e689446886f3cee13de44fec8c02f","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"1":"{\"text\":\"normal distribution\\n\"}"}],"duration":2}},{"eid":"ASSESS","ets":1569304761105,"ver":"3.1","mid":"ASSESS:0cba50019b880fc064a343a6d01d3d1a","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"binomial distribution\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1569304763323,"ver":"3.1","mid":"ASSESS:ace9f977fdb8254e709180777ff81ba6","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"4":"{\"text\":\"uniform distribution\\n\"}"}],"duration":6}},{"eid":"ASSESS","ets":1569304765515,"ver":"3.1","mid":"ASSESS:b4b19e35aed5b32cf240650aa09ec558","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"3":"{\"text\":\"Poisson distribution\\n\"}"}],"duration":8}},{"eid":"ASSESS","ets":1569304767576,"ver":"3.1","mid":"ASSESS:366e140cf5fddf850d1e548644a35729","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0,"resvalues":[{"3":"{\"text\":\"variance\\n\"}"}],"duration":1}},{"eid":"ASSESS","ets":1569304769433,"ver":"3.1","mid":"ASSESS:834b83c5205b657a208e552d5964a872","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"Yes","score":1,"resvalues":[{"4":"{\"text\":\"random variable\\n\"}"}],"duration":3}}]}"""
 
-    val LATEST_BATCH_ASSESS_EVENT ="""{"assessmentTs":1587988642000,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":1,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8756","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362dr776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":1,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val QUESTION_EVENT_RES_VALUES = """{"assessmentTs":1578663044164,"userId":"50a9e3fc-d047-4fa5-a37b-67501b8933db","contentId":"do_3129323935897108481169","courseId":"do_3129323995959541761169","batchId":"0129324118211215362","attemptId":"702ae8b81e37c94448d1fa117678d68c","events":[{"eid":"ASSESS","ets":1578663044164,"ver":"3.0","mid":"ASSESS:0b5c1d06920337df9d662b69d53f48bf","actor":{"id":"50a9e3fc-d047-4fa5-a37b-67501b8933db","type":"User"},"context":{"channel":"0126684405014528002","pdata":{"id":"prod.diksha.app","ver":"2.6.203","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"11059dcc-0a69-4d92-90b3-c1eb97f5f93b","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[{"id":"83f8c374dc0e4d91ec4c48d6e5d10b4c","type":"AttemptId"},{"id":"streaming","type":"PlayerLaunch"},{"id":"6a7809b4eccf75ae62b18c224a4cfeb5","type":"ContentSession"},{"id":"48e8e20e8c6fbc660f4ca317497fa7a6","type":"PlaySession"}],"rollup":{"l1":"0126684405014528002"}},"object":{"id":"do_3129323935897108481169","type":"Content","ver":"1","rollup":{"l1":"do_3129323995959541761169","l2":"do_31293239986759270411495"}},"tags":[],"edata":{"item":{"id":"QMCQ02115","maxscore":1,"exlength":0,"params":[{"item":{"keywords":["mdd"],"subject":"Mathematics","channel":"in.ekstep","language":["English"],"type":"ftb","editorState":null,"body":null,"question_audio":"","gradeLevel":["Class 4"],"appId":"prod.diksha.app","used_for":"worksheet","model":{"hintMsg":"Observe the key value and find the number of aircrafts which flies in the morning.","numericLangId":"en","langId":"en","variables":{"$aircraft1":"random(100,100)","$aircraft2":"random(70,70)","$aircraft3":"random(80,80)","$aircraft4":"random(30,30)"},"dropDowns":[{"identifier":1,"options":[{"text":"OPT_1_0","mh":null,"mmc":["DH277","DH278"],"answer":true,"image":null},{"text":"OPT_1_1","mh":"MH_1_1","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_2","mh":"MH_1_2","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_3","mh":"MH_1_3","mmc":["DH277","DH278"],"answer":false,"image":null}]},{"identifier":2,"options":[]},{"identifier":3,"options":[]},{"identifier":4,"options":[]}],"variablesProcessed":true},"state":"Verified","identifier":"QMCQ02115","level":2,"author":"funtoot","consumerId":"1762141d-8681-458b-9bd1-c6af98c0d989","solutions":null,"portalOwner":"562","version":1,"i18n":{"en":{"HINT":"Hint","MICROHINT":"Micro Hint","SOLUTION":"Solution","HELP":"Help","REDUCE_FRACTION":"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.","MIXED_FRACTION":"Convert the mixed fraction to an improper fraction.","IMPROPER_FRACTION":"Convert the improper fraction to an mixed fraction.","LIKEFRACTIONADDITION_NUMERATOR":"Add the numerator of the given fractions correctly.","LIKEFRACTIONADDITION_DENOMINATOR":"For addition of like fractions the denominator remains the same.","LIKEFRACTIONADDITION_FULL":"Check the addition of the numerator. The denominator remains the same.","UNLIKEFRACTIONADDITION_NUMERATOR":"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.","UNLIKEFRACTIONADDITION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONADDITION_FULL":"Convert the given fractions to like fractions and then carefully do the addition.","LIKEFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the given fractions correctly.","LIKEFRACTIONSUBTRACTION_DENOMINATOR":"For subtraction of like fractions the denominator remains the same as that of the question.","LIKEFRACTIONSUBTRACTION_FULL":"Subtract the numerator of the fractions. Denominator remains the same.","UNLIKEFRACTIONSUBTRACTION_NUMERATOR":"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_FULL":"Convert the given fractions to like fractions and then carefully do the subtraction.","LIKEMIXEDFRACTIONADDITION_WHOLE":"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.","LIKEMIXEDFRACTIONADDITION_NUMERATOR":"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.","LIKEMIXEDFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONADDITION_FULL":"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.","LIKEMIXEDFRACTIONSUBTRACTION_WHOLE":"Subtract the whole number part of both the fractions carefully.","LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_FULL":"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_WHOLE":"The integer added becomes the whole number part of the mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_NUMERATOR":"The numerator of the solution fraction remains the same. ","WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem.","WHOLEANDPROPERFRACTIONADDITION_FULL":"The integer and the proper fraction when added becomes the mixed fraction.","WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE":"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.","WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR":"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.","WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem. ","WHOLEANDPROPERFRACTIONSUBTRACTION_FULL":"When the proper fraction is subtracted from the integer becomes the mixed fraction.","WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerator and the whole number carefully.","WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","WHOLEANDFRACTIONMULTIPLICATION_FULL":"Multiply the integer with the fraction.","FRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerators of the fractions carefully.","FRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","FRACTIONMULTIPLICATION_FULL":"Multiply the numerators and denominators of the fractions.","FRACTIONDIVISION":"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.","SELECT":"Select","EXPRESSIONS":"$aircraft1=random(100,100)\n$aircraft2=random(70,70)\n$aircraft3=random(80,80)\n$aircraft4=random(30,30)","NO_HINT":"There is no hint for this question.","NO_ANSWER":"Please answer.","SOLUTION_ID":"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double  that of Mathematics. Hence the statement is true.","HINT_ID":"Observe the key value and find the number of aircrafts which flies in the morning.","QUESTION_TEXT":"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__","MH_1_3":"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_3":"$aircraft1","MH_1_2":"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_2":"$aircraft3","MH_1_1":"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_1":"$aircraft2","OPT_1_0":"$aircraft4"}},"tags":["mdd"],"concepts":["C421"],"grade":["4"],"domain":"Numeracy","name":"QMCQ02115","bloomsTaxonomyLevel":"Understand","status":"Live","itemType":"UNIT","code":"QMCQ02115","qtype":"mdd","qlevel":"MEDIUM","flags":"{\"isZoomable\":true}","questionImage":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","media":[{"id":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","src":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png","type":"image"}],"title":"","qid":"QMCQ02115","createdOn":"2017-12-24T05:10:03.956+0000","qindex":1,"lastUpdatedOn":"2018-08-16T14:08:14.323+0000","subLevel":"","question":"QUESTION_TEXT","versionKey":"1534428494323","framework":"NCF","answer":{},"max_score":5,"sublevel":3,"template_id":"org.ekstep.funtoot.template.01","category":"MCQ","isSelected":true,"$$hashKey":"object:1540","template":"funtoot.template.01","maxAttempts":2,"curAttempt":0},"config":{"count":1,"selectedConfig":{},"title":"","type":"items","var":"item"}}],"uri":"","mmc":[],"mc":[null],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"random(30,30)"}],"duration":6}}]}"""
 
-    val BATCH_ASSESS__OLDER_EVENT: String = """{"courseId":"do_2128415652377067521125","batchId":"012846671379595264119","contentId":"do_212686723743318016173","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","assessmentTs":1567073236195,"attemptId":"8cd87e24df268ad09a8b0060c0a40271","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":1567073236195,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}"""
+  // Cache Does not have leafnodes for this course // is_valid content true
+  val INVALID_CONTENT_ID_EVENT= """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125-invalid","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_212686723743318016173-invalid","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_212686723743318016173","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
 
-    val BATCH_ASSESS_FAIL_EVENT: String = """{"courseId":"do_312712196780204032110117","batchId":"01271220181664563270","contentId":"505c7c48ac6dc1edc9b08f21db5a571d","userId":"b3541347e18ab916c06ed76aeb0ce57f","assessmentTs":1567073236195,"attemptId":"attempt1","events":[{"eid":"ASSESS","ets":1567687351000,"ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":10,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":1,"pass":"Yes","score":5,"resvalues":[{"∑χ":"true"}],"duration":9}},{"eid":"ASSESS","ets":"1567073236195","ver":"3.0","mid":"ASSESS:48d754770446994b98e64577683ada25","actor":{"id":"b3541347e18ab916c06ed76aeb0ce57f","type":"User"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.diksha.portal","ver":"2.2.1","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"310f123e-2764-c784-803a-0ca871f2b651","did":"b3541347e18ab916c06ed76aeb0ce57f","cdata":[{"id":"6b8617ccf0b4c35f3fe17ccdb71af908","type":"ContentSession"},{"id":"01271220181664563270","type":"batch"},{"id":"do_312712196780204032110117","type":"course"}],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_312592741863645184122936","type":"Content","ver":"3","rollup":{"l1":"do_3126430145280819201402"}},"tags":[],"edata":{"item":{"id":"do_312468066279276544217372","maxscore":1,"exlength":0,"params":[],"uri":"","title":"TNXMATHS-STATISTICS 5","mmc":[],"mc":[],"desc":"For any collection of n  items  ∑(χ - χ̅  )="},"index":2,"pass":"No","score":0,"resvalues":[{"∑χ":"true"}],"duration":9}}]}"""
+  //Recompute aggregate event
+  val RECOMPUTE_ASSESS_EVENT = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_2128415652377067521125","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","contentId":"do_212686723743318016173"}"""
 
-    val BATCH_DUPLICATE_QUESTION_EVENT: String = """{"assessmentTs":1569304757079,"batchId":"01284169026368307244","courseId":"do_2128410273679114241112","userId":"d0d8a341-9637-484c-b871-0c27015af238","attemptId":"90e1a0d12542806389a1a52aaf1fc622","contentId":"do_2128373396098744321673","events":[{"eid":"ASSESS","ets":1569304758743,"ver":"3.1","mid":"ASSESS:5b2e689446886f3cee13de44fec8c02f","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"1":"{\"text\":\"normal distribution\\n\"}"}],"duration":2}},{"eid":"ASSESS","ets":1569304761105,"ver":"3.1","mid":"ASSESS:0cba50019b880fc064a343a6d01d3d1a","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"binomial distribution\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1569304763323,"ver":"3.1","mid":"ASSESS:ace9f977fdb8254e709180777ff81ba6","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"4":"{\"text\":\"uniform distribution\\n\"}"}],"duration":6}},{"eid":"ASSESS","ets":1569304765515,"ver":"3.1","mid":"ASSESS:b4b19e35aed5b32cf240650aa09ec558","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"b7886294-95dd-4c83-91e3-7b67e82aaab2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"normal distribution\\n\"}"},{"2":"{\"text\":\"binomial distribution\\n\"}"},{"3":"{\"text\":\"Poisson distribution\\n\"}"},{"4":"{\"text\":\"uniform distribution\\n\"}"},{"answer":"{\"correct\":[\"3\"]}"}],"uri":"","title":"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"3":"{\"text\":\"Poisson distribution\\n\"}"}],"duration":8}},{"eid":"ASSESS","ets":1569304767576,"ver":"3.1","mid":"ASSESS:366e140cf5fddf850d1e548644a35729","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0,"resvalues":[{"3":"{\"text\":\"variance\\n\"}"}],"duration":1}},{"eid":"ASSESS","ets":1569304769433,"ver":"3.1","mid":"ASSESS:834b83c5205b657a208e552d5964a872","actor":{"id":"d0d8a341-9637-484c-b871-0c27015af238","type":"User"},"context":{"channel":"0124511394914140160","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw","did":"609b1be929adff933abd1b32caf10b6d","cdata":[{"id":"do_2128410273679114241112","type":"course"},{"type":"batch","id":"01284169026368307244"},{"id":"f16a4cacf105ca65e98c61f5a63b8bd3","type":"ContentSession"}],"rollup":{"l1":"0124511394914140160","l2":"01245115225042944040"}},"object":{"id":"do_2128373396098744321673","type":"Content","ver":"1","rollup":{"l1":"do_2128410273679114241112","l2":"do_2128410274404106241113"}},"tags":["0124511394914140160","01245115225042944040"],"edata":{"item":{"id":"759b8138-4c47-4724-8cd6-cae6a7d1ad18","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"descriptive statistic\\n\"}"},{"2":"{\"text\":\"probability function\\n\"}"},{"3":"{\"text\":\"variance\\n\"}"},{"4":"{\"text\":\"random variable\\n\"}"},{"answer":"{\"correct\":[\"4\"]}"}],"uri":"","title":"A numerical description of the outcome of an experiment is called a\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"Yes","score":1,"resvalues":[{"4":"{\"text\":\"random variable\\n\"}"}],"duration":3}}]}"""
+  val courseLeafNodes_1 = Map("do_2128415652377067521125:do_2128415652377067521125:leafnodes" -> "do_212686723743318016173")
+  val courseLeafNodes_2 = Map("do_312712196780204032110117:do_312712196780204032110117:leafnodes" -> "505c7c48ac6dc1edc9b08f21db5a571d")
+  val courseLeafNodes_3 = Map("do_2128410273679114241112:do_2128410273679114241112:leafnodes" -> "do_2128373396098744321673")
+  val courseLeafNodes_4 = Map("do_3129323995959541761169:do_3129323995959541761169:leafnodes" -> "do_3129323935897108481169")
+  val courseLeafNodes_5 = Map("do_873264782364827482:do_873264782364827482:leafnodes" -> "do_313026415363981312122")
+  val courseLeafNodes_6 = Map("do_87326478236482748244:do_87326478236482748244:leafnodes" -> "do_1131998128479272961991")
+  val leafNodesList = List(courseLeafNodes_1, courseLeafNodes_2, courseLeafNodes_3, courseLeafNodes_4, courseLeafNodes_6)
 
-    val QUESTION_EVENT_RES_VALUES = """{"assessmentTs":1578663044164,"userId":"50a9e3fc-d047-4fa5-a37b-67501b8933db","contentId":"do_3129323935897108481169","courseId":"do_3129323995959541761169","batchId":"0129324118211215362","attemptId":"702ae8b81e37c94448d1fa117678d68c","events":[{"eid":"ASSESS","ets":1578663044164,"ver":"3.0","mid":"ASSESS:0b5c1d06920337df9d662b69d53f48bf","actor":{"id":"50a9e3fc-d047-4fa5-a37b-67501b8933db","type":"User"},"context":{"channel":"0126684405014528002","pdata":{"id":"prod.diksha.app","ver":"2.6.203","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"11059dcc-0a69-4d92-90b3-c1eb97f5f93b","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[{"id":"83f8c374dc0e4d91ec4c48d6e5d10b4c","type":"AttemptId"},{"id":"streaming","type":"PlayerLaunch"},{"id":"6a7809b4eccf75ae62b18c224a4cfeb5","type":"ContentSession"},{"id":"48e8e20e8c6fbc660f4ca317497fa7a6","type":"PlaySession"}],"rollup":{"l1":"0126684405014528002"}},"object":{"id":"do_3129323935897108481169","type":"Content","ver":"1","rollup":{"l1":"do_3129323995959541761169","l2":"do_31293239986759270411495"}},"tags":[],"edata":{"item":{"id":"QMCQ02115","maxscore":1,"exlength":0,"params":[{"item":{"keywords":["mdd"],"subject":"Mathematics","channel":"in.ekstep","language":["English"],"type":"ftb","editorState":null,"body":null,"question_audio":"","gradeLevel":["Class 4"],"appId":"prod.diksha.app","used_for":"worksheet","model":{"hintMsg":"Observe the key value and find the number of aircrafts which flies in the morning.","numericLangId":"en","langId":"en","variables":{"$aircraft1":"random(100,100)","$aircraft2":"random(70,70)","$aircraft3":"random(80,80)","$aircraft4":"random(30,30)"},"dropDowns":[{"identifier":1,"options":[{"text":"OPT_1_0","mh":null,"mmc":["DH277","DH278"],"answer":true,"image":null},{"text":"OPT_1_1","mh":"MH_1_1","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_2","mh":"MH_1_2","mmc":["DH277","DH278"],"answer":false,"image":null},{"text":"OPT_1_3","mh":"MH_1_3","mmc":["DH277","DH278"],"answer":false,"image":null}]},{"identifier":2,"options":[]},{"identifier":3,"options":[]},{"identifier":4,"options":[]}],"variablesProcessed":true},"state":"Verified","identifier":"QMCQ02115","level":2,"author":"funtoot","consumerId":"1762141d-8681-458b-9bd1-c6af98c0d989","solutions":null,"portalOwner":"562","version":1,"i18n":{"en":{"HINT":"Hint","MICROHINT":"Micro Hint","SOLUTION":"Solution","HELP":"Help","REDUCE_FRACTION":"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.","MIXED_FRACTION":"Convert the mixed fraction to an improper fraction.","IMPROPER_FRACTION":"Convert the improper fraction to an mixed fraction.","LIKEFRACTIONADDITION_NUMERATOR":"Add the numerator of the given fractions correctly.","LIKEFRACTIONADDITION_DENOMINATOR":"For addition of like fractions the denominator remains the same.","LIKEFRACTIONADDITION_FULL":"Check the addition of the numerator. The denominator remains the same.","UNLIKEFRACTIONADDITION_NUMERATOR":"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.","UNLIKEFRACTIONADDITION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONADDITION_FULL":"Convert the given fractions to like fractions and then carefully do the addition.","LIKEFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the given fractions correctly.","LIKEFRACTIONSUBTRACTION_DENOMINATOR":"For subtraction of like fractions the denominator remains the same as that of the question.","LIKEFRACTIONSUBTRACTION_FULL":"Subtract the numerator of the fractions. Denominator remains the same.","UNLIKEFRACTIONSUBTRACTION_NUMERATOR":"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_DENOMINATOR":"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.","UNLIKEFRACTIONSUBTRACTION_FULL":"Convert the given fractions to like fractions and then carefully do the subtraction.","LIKEMIXEDFRACTIONADDITION_WHOLE":"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.","LIKEMIXEDFRACTIONADDITION_NUMERATOR":"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.","LIKEMIXEDFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONADDITION_FULL":"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.","LIKEMIXEDFRACTIONSUBTRACTION_WHOLE":"Subtract the whole number part of both the fractions carefully.","LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR":"Subtract the numerator of the like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction does not change for like fractions.","LIKEMIXEDFRACTIONSUBTRACTION_FULL":"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_WHOLE":"The integer added becomes the whole number part of the mixed fraction.","WHOLEANDPROPERFRACTIONADDITION_NUMERATOR":"The numerator of the solution fraction remains the same. ","WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem.","WHOLEANDPROPERFRACTIONADDITION_FULL":"The integer and the proper fraction when added becomes the mixed fraction.","WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE":"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.","WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR":"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.","WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR":"The denominator of the solution fraction remains the same as given in the problem. ","WHOLEANDPROPERFRACTIONSUBTRACTION_FULL":"When the proper fraction is subtracted from the integer becomes the mixed fraction.","WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerator and the whole number carefully.","WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","WHOLEANDFRACTIONMULTIPLICATION_FULL":"Multiply the integer with the fraction.","FRACTIONMULTIPLICATION_NUMERATOR":"Multiply the numerators of the fractions carefully.","FRACTIONMULTIPLICATION_DENOMINATOR":"Multiply the denominators of the fractions carefully. ","FRACTIONMULTIPLICATION_FULL":"Multiply the numerators and denominators of the fractions.","FRACTIONDIVISION":"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.","SELECT":"Select","EXPRESSIONS":"$aircraft1=random(100,100)\n$aircraft2=random(70,70)\n$aircraft3=random(80,80)\n$aircraft4=random(30,30)","NO_HINT":"There is no hint for this question.","NO_ANSWER":"Please answer.","SOLUTION_ID":"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double  that of Mathematics. Hence the statement is true.","HINT_ID":"Observe the key value and find the number of aircrafts which flies in the morning.","QUESTION_TEXT":"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__","MH_1_3":"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_3":"$aircraft1","MH_1_2":"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_2":"$aircraft3","MH_1_1":"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.","OPT_1_1":"$aircraft2","OPT_1_0":"$aircraft4"}},"tags":["mdd"],"concepts":["C421"],"grade":["4"],"domain":"Numeracy","name":"QMCQ02115","bloomsTaxonomyLevel":"Understand","status":"Live","itemType":"UNIT","code":"QMCQ02115","qtype":"mdd","qlevel":"MEDIUM","flags":"{\"isZoomable\":true}","questionImage":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","media":[{"id":"org.ekstep.funtoot.QMCQ02115.image.9074280266192281","src":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png","type":"image"}],"title":"","qid":"QMCQ02115","createdOn":"2017-12-24T05:10:03.956+0000","qindex":1,"lastUpdatedOn":"2018-08-16T14:08:14.323+0000","subLevel":"","question":"QUESTION_TEXT","versionKey":"1534428494323","framework":"NCF","answer":{},"max_score":5,"sublevel":3,"template_id":"org.ekstep.funtoot.template.01","category":"MCQ","isSelected":true,"$$hashKey":"object:1540","template":"funtoot.template.01","maxAttempts":2,"curAttempt":0},"config":{"count":1,"selectedConfig":{},"title":"","type":"items","var":"item"}}],"uri":"","mmc":[],"mc":[null],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"random(30,30)"}],"duration":6}}]}"""
-}
+  val DUPLICATE_BATCH_ASSESS_EVENTS_1  = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_313026415363981312122","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val DUPLICATE_BATCH_ASSESS_EVENTS_2 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_11307972307046400011917","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_1","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_2","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776_3","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_11307972307046400011917","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val DUPLICATE_BATCH_ASSESS_EVENTS_3 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_873264782364827482","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_313026415363981312122123","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val DUPLICATE_BATCH_ASSESS_EVENTS_4 = """{"assessmentTs":1568891729576,"batchId":"012846671379595264119","courseId":"do_87326478236482748244","userId":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","attemptId":"8cd87e24df268ad09a8b0060c0a40271","contentId":"do_1131998128479272961991","events":[{"eid":"ASSESS","ets":1568891735461,"ver":"3.1","mid":"ASSESS:db00a858fec1b8796c62f224874c7edf","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[],"duration":2}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1568891747395,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1568891772964,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}},{"eid":"ASSESS","ets":1568891738245,"ver":"3.1","mid":"ASSESS:135815023ec32a430632ba5d7f84fe18","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"No","score":0,"resvalues":[{"2":"{\"text\":\"Work Heavy Organization\\n\"}"}],"duration":4}},{"eid":"ASSESS","ets":1626595233000,"ver":"3.1","mid":"ASSESS:6ba5953669ea86e8f85759d3e7f5998b","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"801ae93c-8807-4be5-8853-dd49362d8776","maxscore":1,"type":"mcq","exlength":0,"params":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"},{"2":"{\"text\":\"Work Heavy Organization\\n\"}"},{"3":"{\"text\":\"Work hell Organization\\n\"}"},{"4":"{\"text\":\"None of The above\\n\"}"},{"answer":"{\"correct\":[\"1\"]}"}],"uri":"","title":"What is the Full form of WHO..?\n","mmc":[],"mc":[],"desc":""},"index":1,"pass":"Yes","score":1,"resvalues":[{"1":"{\"text\":\"World Health Organizaton\\n\"}"}],"duration":14}},{"eid":"ASSESS","ets":1629273633000,"ver":"3.1","mid":"ASSESS:018f01bf99288474860b630b513b9d0c","actor":{"id":"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f","type":"User"},"context":{"channel":"0124784842112040965","pdata":{"id":"staging.diksha.portal","ver":"2.4.0","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe","did":"a08946e8b72abfeeff6642f245d470cb","cdata":[{"id":"do_2128415652377067521127","type":"course"},{"type":"batch","id":"012846671379595264119"},{"id":"f3ec2acf4360e93172b9234e29e38be4","type":"ContentSession"}],"rollup":{"l1":"0124784842112040965"}},"object":{"id":"do_313026415363981312122","type":"Content","ver":"1","rollup":{"l1":"do_2128415652377067521127","l2":"do_2128415660716359681128"}},"tags":["0124784842112040965"],"edata":{"item":{"id":"2bc922e7-985e-486a-ae23-4ba9a1c67edc","maxscore":1,"type":"mtf","exlength":0,"params":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"},{"answer":"{\"lhs\":[\"1\",\"2\",\"3\"],\"rhs\":[\"3\",\"1\",\"2\"]}"}],"uri":"","title":"MTF 3\n","mmc":[],"mc":[],"desc":""},"index":2,"pass":"No","score":0.33,"resvalues":[{"lhs":"[{\"1\":\"{\\\"text\\\":\\\"1\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"3\\\"}\"}]"},{"rhs":"[{\"1\":\"{\\\"text\\\":\\\"3\\\"}\"},{\"2\":\"{\\\"text\\\":\\\"2\\\"}\"},{\"3\":\"{\\\"text\\\":\\\"1\\\"}\"}]"}],"duration":24}}]}"""
+  val contentCacheData_1 = Map("do_313026415363981312122" -> "{\n            \"ownershipType\": [\n                \"createdBy\"\n            ],\n            \"copyright\": \"EKSTEP\",\n            \"previewUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n            \"plugins\": [\n                {\n                    \"identifier\": \"org.ekstep.stage\",\n                    \"semanticVersion\": \"1.0\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionset\",\n                    \"semanticVersion\": \"1.0\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.navigation\",\n                    \"semanticVersion\": \"1.0\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit\",\n                    \"semanticVersion\": \"1.1\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit.reorder\",\n                    \"semanticVersion\": \"1.1\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit.sequence\",\n                    \"semanticVersion\": \"1.1\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionset.quiz\",\n                    \"semanticVersion\": \"1.0\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.iterator\",\n                    \"semanticVersion\": \"1.0\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.keyboard\",\n                    \"semanticVersion\": \"1.1\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit.ftb\",\n                    \"semanticVersion\": \"1.1\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit.mtf\",\n                    \"semanticVersion\": \"1.2\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.questionunit.mcq\",\n                    \"semanticVersion\": \"1.3\"\n                },\n                {\n                    \"identifier\": \"org.ekstep.summary\",\n                    \"semanticVersion\": \"1.0\"\n                }\n            ],\n            \"subject\": [\n                \"CBSE Training\"\n            ],\n            \"channel\": \"0123221758376673287017\",\n            \"downloadUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n            \"questions\": [\n                {\n                    \"name\": \"Arrange the Given sentence \\n\",\n                    \"relation\": \"associatedTo\",\n                    \"identifier\": \"do_313026448412631040111\",\n                    \"description\": null,\n                    \"objectType\": \"AssessmentItem\",\n                    \"status\": \"Live\"\n                },\n                {\n                    \"name\": \"Test data mark is 10\\n\",\n                    \"relation\": \"associatedTo\",\n                    \"identifier\": \"do_313026416457089024115\",\n                    \"description\": null,\n                    \"objectType\": \"AssessmentItem\",\n                    \"status\": \"Live\"\n                },\n                {\n                    \"name\": \"Arrange the given words in proper sentence\\n\",\n                    \"relation\": \"associatedTo\",\n                    \"identifier\": \"do_313026446885158912120\",\n                    \"description\": null,\n                    \"objectType\": \"AssessmentItem\",\n                    \"status\": \"Live\"\n                },\n                {\n                    \"name\": \"'The tree ____ is ____ and ____ mark is 3\\n\",\n                    \"relation\": \"associatedTo\",\n                    \"identifier\": \"do_313026423737737216117\",\n                    \"description\": null,\n                    \"objectType\": \"AssessmentItem\",\n                    \"status\": \"Live\"\n                },\n                {\n                    \"name\": \"Match the following\\n\",\n                    \"relation\": \"associatedTo\",\n                    \"identifier\": \"do_31302641810772787218\",\n                    \"description\": null,\n                    \"objectType\": \"AssessmentItem\",\n                    \"status\": \"Live\"\n                }\n            ],\n            \"organisation\": [\n                \"EKSTEP\"\n            ],\n            \"language\": [\n                \"English\"\n            ],\n            \"mimeType\": \"application/vnd.ekstep.ecml-archive\",\n            \"variants\": {\n                \"spine\": {\n                    \"ecarUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_313026415363981312122/vjjjj_1590142788484_do_313026415363981312122_1.0_spine.ecar\",\n                    \"size\": 44094\n                }\n            },\n            \"editorState\": \"{\\\"plugin\\\":{\\\"noOfExtPlugins\\\":11,\\\"extPlugins\\\":[{\\\"plugin\\\":\\\"org.ekstep.contenteditorfunctions\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboardshortcuts\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.richtext\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.iterator\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.navigation\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.reviewercomments\\\",\\\"version\\\":\\\"1.0\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mtf\\\",\\\"version\\\":\\\"1.2\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.mcq\\\",\\\"version\\\":\\\"1.3\\\"},{\\\"plugin\\\":\\\"org.ekstep.keyboard\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.reorder\\\",\\\"version\\\":\\\"1.1\\\"},{\\\"plugin\\\":\\\"org.ekstep.questionunit.sequence\\\",\\\"version\\\":\\\"1.1\\\"}]},\\\"stage\\\":{\\\"noOfStages\\\":1,\\\"currentStage\\\":\\\"371a0b37-e8ab-4e8e-b83e-3775a2ed927d\\\",\\\"selectedPluginObject\\\":\\\"49469445-2ca2-479d-95b7-360aa07ee3bd\\\"},\\\"sidebar\\\":{\\\"selectedMenu\\\":\\\"settings\\\"}}\",\n            \"objectType\": \"Content\",\n            \"gradeLevel\": [\n                \"Class 1\"\n            ],\n            \"appIcon\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/10_1560927487910.thumb.jpg\",\n            \"primaryCategory\": \"Course Assessment\",\n            \"collections\": [\n                {\n                    \"name\": \"Copy of SelfServiceable\",\n                    \"relation\": \"hasSequenceMember\",\n                    \"identifier\": \"do_313026450799894528150\",\n                    \"description\": \"Execution\",\n                    \"objectType\": \"Collection\",\n                    \"status\": \"Retired\"\n                }\n            ],\n            \"appId\": \"prod.diksha.portal\",\n            \"contentEncoding\": \"gzip\",\n            \"artifactUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_313026415363981312122/artifact/1590142788151_do_313026415363981312122.zip\",\n            \"lockKey\": \"0708a132-4774-4126-b2e5-df0e83f6dd40\",\n            \"sYS_INTERNAL_LAST_UPDATED_ON\": \"2020-05-22T10:19:50.208+0000\",\n            \"contentType\": \"SelfAssess\",\n            \"lastUpdatedBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n            \"identifier\": \"do_313026415363981312122\",\n            \"audience\": [\n                \"Teacher\"\n            ],\n            \"visibility\": \"Default\",\n            \"consumerId\": \"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\",\n            \"mediaType\": \"content\",\n            \"osId\": \"org.ekstep.quiz.app\",\n            \"lastPublishedBy\": \"5c627fa2-a7a4-490f-b9c8-bc0f42559562\",\n            \"languageCode\": [\n                \"en\"\n            ],\n            \"version\": 2,\n            \"license\": \"CC BY 4.0\",\n            \"prevState\": \"Review\",\n            \"size\": 547518,\n            \"lastPublishedOn\": \"2020-05-22T10:19:48.378+0000\",\n            \"name\": \"Vjjjj\",\n            \"status\": \"Retired\",\n            \"totalQuestions\": 2,\n            \"code\": \"org.sunbird.lfSsbq\",\n            \"prevStatus\": \"Processing\",\n            \"description\": \"Enter description for Assessment\",\n            \"streamingUrl\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/ecml/do_313026415363981312122-latest\",\n            \"medium\": [\n                \"English\"\n            ],\n            \"posterImage\": \"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_3127871179775098881294/artifact/10_1560927487910.jpg\",\n            \"idealScreenSize\": \"normal\",\n            \"createdOn\": \"2020-05-22T09:09:54.237+0000\",\n            \"copyrightYear\": 2020,\n            \"contentDisposition\": \"inline\",\n            \"licenseterms\": \"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.\",\n            \"lastUpdatedOn\": \"2020-05-22T11:59:41.479+0000\",\n            \"dialcodeRequired\": \"No\",\n            \"lastStatusChangedOn\": \"2020-05-22T11:59:41.479+0000\",\n            \"createdFor\": [\n                \"0123221758376673287017\"\n            ],\n            \"creator\": \"content_creator sahu\",\n            \"os\": [\n                \"All\"\n            ],\n            \"totalScore\": 5,\n            \"pkgVersion\": 1,\n            \"versionKey\": \"1590142785805\",\n            \"idealScreenDensity\": \"hdpi\",\n            \"framework\": \"ekstep_ncert_k-12\",\n            \"s3Key\": \"ecar_files/do_313026415363981312122/vjjjj_1590142788385_do_313026415363981312122_1.0.ecar\",\n            \"lastSubmittedOn\": \"2020-05-22T10:19:07.606+0000\",\n            \"createdBy\": \"2dded955-5656-4dc2-a683-e7e42082aa8c\",\n            \"compatibilityLevel\": 2,\n            \"board\": \"CBSE\"\n        }")
+  val contentCacheData_2 = Map("do_313026415363981312122123" -> "{\"totalQuestions\":1}")
+  val contentCacheList = List(contentCacheData_1, contentCacheData_2)
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
index 5e25f62abd..3cea7b170b 100644
--- a/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
+++ b/data-pipeline-flink/assessment-aggregator/src/test/scala/org/sunbird/dp/spec/AssessmentAggregatorTaskTestSpec.scala
@@ -1,9 +1,10 @@
 package org.sunbird.dp.spec
 
-import java.util
+import com.datastax.driver.core.Row
 
+import java.util
+import com.google.common.reflect.TypeToken
 import com.google.gson.Gson
-import com.google.gson.internal.LinkedTreeMap
 import com.typesafe.config.{Config, ConfigFactory}
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
@@ -18,125 +19,270 @@ import org.cassandraunit.utils.EmbeddedCassandraServerHelper
 import org.mockito.Mockito
 import org.mockito.Mockito._
 import org.sunbird.dp.assessment.domain.Event
+import org.sunbird.dp.assessment.functions.{AggDetails, UserActivityAgg}
 import org.sunbird.dp.assessment.task.{AssessmentAggregatorConfig, AssessmentAggregatorStreamTask}
+import org.sunbird.dp.core.cache.RedisConnect
 import org.sunbird.dp.core.job.FlinkKafkaConnector
-import org.sunbird.dp.core.util.CassandraUtil
+import org.sunbird.dp.core.util.{CassandraUtil, JSONUtil}
 import org.sunbird.dp.fixture.EventFixture
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
-import org.junit.Assert.{assertEquals, assertNotNull}
+import redis.embedded.RedisServer
+import okhttp3.mockwebserver.{MockResponse, MockWebServer}
+import java.io.IOException
 
-import collection.JavaConverters._
 
 class AssessmentAggregatorTaskTestSpec extends BaseTestSpec {
 
-    implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]])
+  implicit val mapTypeInfo: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]])
 
-    val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
-      .setConfiguration(testConfiguration())
-      .setNumberSlotsPerTaskManager(1)
-      .setNumberTaskManagers(1)
-      .build)
+  val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
+    .setConfiguration(testConfiguration())
+    .setNumberSlotsPerTaskManager(1)
+    .setNumberTaskManagers(1)
+    .build)
+  var redisServer: RedisServer = _
 
+  val config: Config = ConfigFactory.load("test.conf")
+  val assessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(config)
+  val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
+  val gson = new Gson()
+  val server = new MockWebServer()
 
-    val config: Config = ConfigFactory.load("test.conf")
-    val assessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(config)
-    val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
-    val gson = new Gson()
 
+  var cassandraUtil: CassandraUtil = _
 
-    var cassandraUtil: CassandraUtil = _
 
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    redisServer = new RedisServer(6340)
+    redisServer.start()
+    EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L)
+    cassandraUtil = new CassandraUtil(assessmentConfig.dbHost, assessmentConfig.dbPort)
+    val session = cassandraUtil.session
+    setupRestUtilData()
+    setupRedisTestData()
 
-    override protected def beforeAll(): Unit = {
-        super.beforeAll()
-        EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L)
-        cassandraUtil = new CassandraUtil(assessmentConfig.dbHost, assessmentConfig.dbPort)
-        val session = cassandraUtil.session
+    val dataLoader = new CQLDataLoader(session)
+    dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true));
+    // Clear the metrics
+    testCassandraUtil(cassandraUtil)
+    BaseMetricsReporter.gaugeMetrics.clear()
 
+    flinkCluster.before()
+  }
 
-        val dataLoader = new CQLDataLoader(session);
-        dataLoader.load(new FileCQLDataSet(getClass.getResource("/test.cql").getPath, true, true));
-        // Clear the metrics
-        testCassandraUtil(cassandraUtil)
-        BaseMetricsReporter.gaugeMetrics.clear()
+  override protected def afterAll(): Unit = {
+    super.afterAll()
+    redisServer.stop()
+    try {
+      EmbeddedCassandraServerHelper.cleanEmbeddedCassandra()
+      server.close()
+    } catch {
+      case ex: Exception => {
 
-        flinkCluster.before()
+      }
     }
+    flinkCluster.after()
+  }
 
-    override protected def afterAll(): Unit = {
-        super.afterAll()
-        try {
-            EmbeddedCassandraServerHelper.cleanEmbeddedCassandra()
-        }catch {
-            case ex: Exception =>{
-
-            }
-        }
-        flinkCluster.after()
+  def setupRestUtilData(): Unit = {
+    val do_11307972307046400011917_response = """{"id":"api.content.read","ver":"1.0","ts":"2023-05-17T10:26:51.549Z","params":{"resmsgid":"566eacd0-f49d-11ed-bf1e-7fae1bdbcdf8","msgid":"566de980-f49d-11ed-8721-d532b5857c8a","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"content":{"ownershipType":["createdBy"],"copyright":"Sunbird Org","previewUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_113762457691021312168/samplevideo_1280x720_1mb.mp4","channel":"0137541424673095687","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660391_do_113762457691021312168_1.ecar","organisation":["Sunbird Org"],"language":["English"],"mimeType":"video/mp4","variants":{"full":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660391_do_113762457691021312168_1.ecar","size":"1058720"},"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/content-2_1679987660892_do_113762457691021312168_1_SPINE.ecar","size":"4153"}},"objectType":"Content","appIcon":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_113762457691021312168/artifact/do_11376182453272576019_1679910221428_287-2876925_test-image-png-unit-testing-png-transparent-png.thumb.png","primaryCategory":"Explanation Content","contentEncoding":"identity","artifactUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/assets/do_113762457691021312168/samplevideo_1280x720_1mb.mp4","lockKey":"1a558750-41dd-43d3-9d4f-9f12184a902e","contentType":"Resource","category2":"Category2 Term1","identifier":"do_113762457691021312168","lastUpdatedBy":"155ce3c5-713e-4749-bc1c-95d09c640914","category3":"Category3 Term1","audience":["Student"],"category4":"Category4 Term1","category5":"Category5 Term1","visibility":"Default","category1":"Category1 Term1","discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"469dc732-04f3-42d9-9a85-30957a797acc","version":2,"license":"CC BY 4.0","prevState":"Review","size":1055736,"lastPublishedOn":"2023-03-28T07:14:20.009+0000","name":"Content - 2","status":"Live","code":"62ada120-13c4-4e94-aad6-56cebe6a089c","interceptionPoints":{},"credentials":{"enabled":"No"},"prevStatus":"Processing","streamingUrl":"https://sunbirdspikemedia-inct.streaming.media.azure.net/5d2643e3-fcae-42a8-8a22-ac291a317ed4/samplevideo_1280x720_1mb.ism/manifest(format=m3u8-aapl-v3)","posterImage":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376182453272576019/artifact/do_11376182453272576019_1679910221428_287-2876925_test-image-png-unit-testing-png-transparent-png.png","idealScreenSize":"normal","createdOn":"2023-03-28T07:11:51.115+0000","contentDisposition":"inline","lastUpdatedOn":"2023-03-28T07:44:16.190+0000","dialcodeRequired":"No","lastStatusChangedOn":"2023-03-28T07:14:21.098+0000","createdFor":["0137541424673095687"],"creator":"contentCreator Creator","os":["All"],"se_FWIds":["NCF"],"pkgVersion":1,"versionKey":"1679989456190","idealScreenDensity":"hdpi","framework":"framework1","lastSubmittedOn":"2023-03-28T07:12:10.623+0000","createdBy":"155ce3c5-713e-4749-bc1c-95d09c640914","compatibilityLevel":1,"resourceType":"Learn"}}}"""
+    val do_1131998128479272961991_response = """{"id":"api.content.read","ver":"1.0","ts":"2023-05-17T11:08:52.532Z","params":{"resmsgid":"350e2740-f4a3-11ed-bf1e-7fae1bdbcdf8","msgid":"350d15d0-f4a3-11ed-8721-d532b5857c8a","status":"successful","err":null,"errmsg":null},"responseCode":"OK","result":{"content":{"ownershipType":["createdBy"],"copyright":"sunbird","se_gradeLevelIds":["ncf_gradelevel_class1"],"subject":["Telugu"],"channel":"0137541424673095687","downloadUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340314_do_11376180991508480011_1_SPINE.ecar","organisation":["Sunbird Org"],"language":["English"],"mimeType":"application/vnd.ekstep.content-collection","variants":{"spine":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340314_do_11376180991508480011_1_SPINE.ecar","size":"8922"},"online":{"ecarUrl":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/test-book_1679910340572_do_11376180991508480011_1_ONLINE.ecar","size":"5179"}},"leafNodes":["do_11376182093890355216","do_11376182438513868818"],"objectType":"Content","se_mediums":["Telugu"],"gradeLevel":["Class 1"],"appIcon":"","primaryCategory":"Digital Textbook","contentEncoding":"gzip","lockKey":"fbeda787-5742-4c45-a535-fbb2c99ac3a0","generateDIALCodes":"Yes","totalCompressedSize":3579342,"mimeTypesCount":"{\"video/mp4\":2,\"application/vnd.ekstep.content-collection\":2}","sYS_INTERNAL_LAST_UPDATED_ON":"2023-03-27T09:45:40.314+0000","contentType":"TextBook","se_gradeLevels":["Class 1"],"trackable":{"enabled":"No","autoBatch":"No"},"identifier":"do_11376180991508480011","audience":["Student"],"se_boardIds":["ncf_board_other"],"subjectIds":["ncf_subject_telugu"],"toc_url":"https://sunbirddev.blob.core.windows.net/sunbird-content-dev/content/do_11376180991508480011/artifact/do_11376180991508480011_toc.json","visibility":"Default","contentTypesCount":"{\"TextBookUnit\":2,\"Resource\":2}","author":"BookCreator bookCreator","consumerId":"bfe5883f-ac66-4744-a064-3ed88d986eba","childNodes":["do_11376182093890355216","do_11376181820567552012","do_11376182438513868818","do_11376181820809216014"],"discussionForum":{"enabled":"No"},"mediaType":"content","osId":"org.ekstep.quiz.app","languageCode":["en"],"lastPublishedBy":"4b4dda54-b061-4346-9aaa-e2801430b885","version":2,"se_subjects":["Telugu"],"license":"CC BY 4.0","prevState":"Review","size":8922,"lastPublishedOn":"2023-03-27T09:45:40.159+0000","name":"Test Book","mediumIds":["ncf_medium_telugu"],"status":"Live","code":"org.sunbird.jMNK3Z","credentials":{"enabled":"No"},"prevStatus":"Processing","description":"Enter description for TextBook","medium":["Telugu"],"idealScreenSize":"normal","createdOn":"2023-03-27T09:13:56.965+0000","se_boards":["Other"],"se_mediumIds":["ncf_medium_telugu"],"copyrightYear":2023,"contentDisposition":"inline","lastUpdatedOn":"2023-03-27T09:45:40.820+0000","dialcodeRequired":"No","lastStatusChangedOn":"2023-03-27T09:45:40.820+0000","createdFor":["0137541424673095687"],"creator":"BookCreator bookCreator","os":["All"],"se_subjectIds":["ncf_subject_telugu"],"se_FWIds":["NCF"],"pkgVersion":1,"versionKey":"1679910320288","idealScreenDensity":"hdpi","framework":"NCF","depth":0,"s3Key":"content/do_11376180991508480011/artifact/do_11376180991508480011_toc.json","boardIds":["ncf_board_other"],"lastSubmittedOn":"2023-03-27T09:45:20.280+0000","createdBy":"7bf81b8b-ab64-47ca-b9d7-c9f74f811980","compatibilityLevel":1,"leafNodesCount":2,"userConsent":"Yes","gradeLevelIds":["ncf_gradelevel_class1"],"board":"Other","resourceType":"Book"}}}"""
+    try {
+      server.start(3000)
+    } catch {
+      case e: IOException =>
+        System.out.println("Exception" + e)
     }
+    server.enqueue(new MockResponse().setBody(do_11307972307046400011917_response))
+    server.url("http://127.0.0.1:3000/api/content/v1/read/do_11307972307046400011917")
+    server.enqueue(new MockResponse().setBody(do_1131998128479272961991_response))
+    server.url("http://127.0.0.1:3000/api/content/v1/read/do_1131998128479272961991")
+  }
 
 
-    "AssessmentAggregator " should "Update event to db" in {
-        when(mockKafkaUtil.kafkaEventSource[Event](assessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSource)
-        when(mockKafkaUtil.kafkaEventSink[Event](assessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink)
-        val task = new AssessmentAggregatorStreamTask(assessmentConfig, mockKafkaUtil)
-        task.process()
-        val failedEvent = gson.fromJson(gson.toJson(FailedEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
-        assert(failedEvent.get("map").get.asInstanceOf[LinkedTreeMap[String, AnyRef]].containsKey("metadata"))
-        BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.skippedEventCount}").getValue() should be(1)
-        BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbReadCount}").getValue() should be(2)
-        BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbUpdateCount}").getValue() should be(4)
-        BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.failedEventCount}").getValue() should be(1)
-        BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(4)
-        val test_row1 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where course_id='do_2128410273679114241112'")
-        assert(test_row1.getDouble("total_score") == 2.0)
-        assert(test_row1.getDouble("total_max_score") == 2.0)
-
-        val test_row2 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where course_id='do_2128415652377067521125'")
-        assert(test_row2.getDouble("total_score") == 3.0)
-        assert(test_row2.getDouble("total_max_score") == 4.0)
-    }
+  "AssessmentAggregator " should "Update event to db" in {
+    when(mockKafkaUtil.kafkaEventSource[Event](assessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSource)
+    when(mockKafkaUtil.kafkaEventSink[Event](assessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink)
+    when(mockKafkaUtil.kafkaStringSink(assessmentConfig.kafkaCertIssueTopic)).thenReturn(new certificateIssuedEventsSink)
+    val task = new AssessmentAggregatorStreamTask(assessmentConfig, mockKafkaUtil)
+    task.process()
+    assert(FailedEventsSink.values.get(0).getTelemetry.getMap.containsKey("metadata"))
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.skippedEventCount}").getValue() should be(1)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbReadCount}").getValue() should be(2)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbUpdateCount}").getValue() should be(6)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.failedEventCount}").getValue() should be(2)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(6)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitCount}").getValue() should be(8)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitMissCount}").getValue() should be(1)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.certIssueEventsCount}").getValue() should be(6)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbScoreAggUpdateCount}").getValue() should be(6)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.dbScoreAggReadCount}").getValue() should be(6)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.recomputeAggEventCount}").getValue() should be(0)
+    val test_row1 = cassandraUtil.findOne("select total_score,total_max_score from sunbird_courses.assessment_aggregator where user_id='d0d8a341-9637-484c-b871-0c27015af238' and course_id='do_2128410273679114241112'")
+    assert(test_row1.getDouble("total_score") == 2.0)
+    assert(test_row1.getDouble("total_max_score") == 2.0)
 
-    def testCassandraUtil(cassandraUtil: CassandraUtil): Unit ={
-        cassandraUtil.reconnect()
-        val response = cassandraUtil.find("SELECT * FROM sunbird_courses.assessment_aggregator;")
-        response should not be(null)
-    }
+    val test_row2: java.util.List[Row] = cassandraUtil.find("select attempt_id,total_score,total_max_score from sunbird_courses.assessment_aggregator where user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f' and course_id='do_2128415652377067521125'")
+
+    assert(test_row2.get(0).getString("attempt_id") == "8cd87e24df268ad09a8b0060c0a40271")
+    assert(test_row2.get(0).getDouble("total_score") == 2.0)
+    assert(test_row2.get(0).getDouble("total_max_score") == 3.0)
+
+    assert(test_row2.get(1).getString("attempt_id") == "9dd87e24df268ad09a8b0060c0a40262")
+    assert(test_row2.get(1).getDouble("total_score") == 1.33)
+    assert(test_row2.get(1).getDouble("total_max_score") == 3.0)
+
+    val test_row3 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128410273679114241112' and  user_id='d0d8a341-9637-484c-b871-0c27015af238'")
+    val resultMap3 = test_row3.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {})
+    assert(null != resultMap3)
+    assert(2 == resultMap3.getOrDefault("score:do_2128373396098744321673", 0))
+    assert(2 == resultMap3.getOrDefault("max_score:do_2128373396098744321673", 0))
+    assert(1 == resultMap3.getOrDefault("attempts_count:do_2128373396098744321673", 0))
+
+    val test_row6 = cassandraUtil.findOne("select agg_details from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128415652377067521125' and  user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f'")
+    val aggDetail1 = new Gson().fromJson(test_row6.getList("agg_details", new TypeToken[String](){}).get(0), classOf[AggDetails])
+    assert(null != aggDetail1)
+    assert("8cd87e24df268ad09a8b0060c0a40271" == aggDetail1.attempt_id)
+    assert("do_212686723743318016173" == aggDetail1.content_id)
+    assert(2 == aggDetail1.score)
+    assert(3 == aggDetail1.max_score)
+    assert("attempt_metrics" == aggDetail1.`type`)
+
+    val aggDetail2 = new Gson().fromJson(test_row6.getList("agg_details", new TypeToken[String](){}).get(1), classOf[AggDetails])
+    assert(null != aggDetail2)
+    assert("9dd87e24df268ad09a8b0060c0a40262" == aggDetail2.attempt_id)
+    assert("do_212686723743318016173" == aggDetail2.content_id)
+    assert(1.33 == aggDetail2.score)
+    assert(3 == aggDetail2.max_score)
+    assert("attempt_metrics" == aggDetail2.`type`)
+
+    val test_row4 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_2128415652377067521125' and  user_id='ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f'")
+    val resultMap4 = test_row4.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {})
+    assert(null != resultMap4)
+    assert(2 == resultMap4.getOrDefault("score:do_212686723743318016173", 0))
+    assert(3 == resultMap4.getOrDefault("max_score:do_212686723743318016173", 0))
+    assert(2 == resultMap4.getOrDefault("attempts_count:do_212686723743318016173", 0))
+
+    val test_row5 = cassandraUtil.findOne("select aggregates from sunbird_courses.user_activity_agg where activity_type='Course' and activity_id='do_3129323995959541761169' and  user_id='50a9e3fc-d047-4fa5-a37b-67501b8933db'")
+    val resultMap5 = test_row5.getMap("aggregates", new TypeToken[String]() {}, new TypeToken[java.lang.Double]() {})
+    assert(null != resultMap5)
+    assert(1 == resultMap5.getOrDefault("score:do_3129323935897108481169", 0))
+    assert(1 == resultMap5.getOrDefault("max_score:do_3129323935897108481169", 0))
+    assert(1 == resultMap5.getOrDefault("attempts_count:do_3129323935897108481169", 0))
+  }
+
+  "AssessmentAggregator " should "Skip the missing records from the event" in {
+    val forceValidationAssessmentConfig: AssessmentAggregatorConfig = new AssessmentAggregatorConfig(ConfigFactory.load("forcevalidate.conf"))
+    when(mockKafkaUtil.kafkaEventSource[Event](forceValidationAssessmentConfig.kafkaInputTopic)).thenReturn(new AssessmentAggreagatorEventSourceForceValidation)
+    when(mockKafkaUtil.kafkaEventSink[Event](forceValidationAssessmentConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink)
+    when(mockKafkaUtil.kafkaStringSink(forceValidationAssessmentConfig.kafkaCertIssueTopic)).thenReturn(new certificateIssuedEventsSink)
+    val task = new AssessmentAggregatorStreamTask(forceValidationAssessmentConfig, mockKafkaUtil)
+    task.process()
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.batchSuccessCount}").getValue() should be(3)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.cacheHitCount}").getValue() should be(5)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.apiHitSuccessCount}").getValue() should be(2)
+    BaseMetricsReporter.gaugeMetrics(s"${assessmentConfig.jobName}.${assessmentConfig.ignoredEventsCount}").getValue() should be(1)
+  }
+
+  def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = {
+    cassandraUtil.reconnect()
+    val response = cassandraUtil.find("SELECT * FROM sunbird_courses.assessment_aggregator;")
+    response should not be (null)
+  }
+
+  def setupRedisTestData() {
+    val redisConnect = new RedisConnect(assessmentConfig.metaRedisHost, assessmentConfig.metaRedisPort, assessmentConfig)
+    val jedis = redisConnect.getConnection(assessmentConfig.relationCacheNode)
+    EventFixture.leafNodesList.map(nodes => {
+      nodes.map(node => {
+        jedis.sadd(node._1, node._2)
+      })
+    })
+
+    // Setup content Cache
+    val contentCache = redisConnect.getConnection(assessmentConfig.contentCacheNode)
+    EventFixture.contentCacheList.map(nodes => {
+      nodes.map(node => {
+        contentCache.set(node._1, node._2)
+      })
+    })
+  }
 }
 
+
+
 class AssessmentAggreagatorEventSource extends SourceFunction[Event] {
 
-    override def run(ctx: SourceContext[Event]) {
-        val gson = new Gson()
-
-        val eventMap1 = gson.fromJson(EventFixture.BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        val eventMap2 = gson.fromJson(EventFixture.BATCH_ASSESS__OLDER_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        val eventMap3 = gson.fromJson(EventFixture.BATCH_ASSESS_FAIL_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        val eventMap4 = gson.fromJson(EventFixture.QUESTION_EVENT_RES_VALUES, new util.LinkedHashMap[String, Any]().getClass)
-        val eventMap5 = gson.fromJson(EventFixture.LATEST_BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        val eventMap6 = gson.fromJson(EventFixture.BATCH_DUPLICATE_QUESTION_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        ctx.collect(new Event(eventMap1))
-        ctx.collect(new Event(eventMap2))
-        ctx.collect(new Event(eventMap3))
-        ctx.collect(new Event(eventMap4))
-        ctx.collect(new Event(eventMap5))
-        ctx.collect(new Event(eventMap6))
-    }
+  override def run(ctx: SourceContext[Event]) {
+    val gson = new Gson()
+
+    //val eventMap1 = gson.fromJson(EventFixture.BATCH_ASSESS_EVENT, new util.LinkedHashMap[String, Any]().getClass)
 
-    override def cancel() = {}
+    val eventMap1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_EVENT)
+    val eventMap2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS__OLDER_EVENT)
+    val eventMap3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_FAIL_EVENT)
+    val eventMap4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.QUESTION_EVENT_RES_VALUES)
+    val eventMap5 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.LATEST_BATCH_ASSESS_EVENT)
+    val eventMap6 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_DUPLICATE_QUESTION_EVENT)
+    val eventMap7 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.INVALID_CONTENT_ID_EVENT)
+    val eventMap8 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.BATCH_ASSESS_EVENT_WITHOUT_CACHE)
+    val eventMap9 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.SECOND_ATTEMPT_BATCH_ASSESS_EVENT)
+    ctx.collect(new Event(eventMap1))
+    ctx.collect(new Event(eventMap2))
+    ctx.collect(new Event(eventMap3))
+    ctx.collect(new Event(eventMap4))
+    ctx.collect(new Event(eventMap5))
+    ctx.collect(new Event(eventMap6))
+    ctx.collect(new Event(eventMap7))
+    ctx.collect(new Event(eventMap8))
+    ctx.collect(new Event(eventMap9))
+  }
+
+  override def cancel() = {}
+
+}
+
+
+class AssessmentAggreagatorEventSourceForceValidation extends SourceFunction[Event] {
+  override def run(ctx: SourceContext[Event]) {
+    val eventMap1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_1)
+    val eventMap2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_2)
+    val eventMap3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_3)
+    val eventMap4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.DUPLICATE_BATCH_ASSESS_EVENTS_4)
+    ctx.collect(new Event(eventMap1))
+    ctx.collect(new Event(eventMap2))
+    ctx.collect(new Event(eventMap3))
+    ctx.collect(new Event(eventMap4))
+  }
+
+  override def cancel() = {}
 
 }
 
 
 class FailedEventsSink extends SinkFunction[Event] {
 
-    override def invoke(value: Event): Unit = {
-        synchronized {
-            FailedEventsSink.values.add(value)
-        }
+  override def invoke(value: Event): Unit = {
+    synchronized {
+      FailedEventsSink.values.add(value)
     }
+  }
 }
 
 object FailedEventsSink {
-    val values: util.List[Event] = new util.ArrayList()
+  val values: util.List[Event] = new util.ArrayList()
+}
+
+class certificateIssuedEventsSink extends SinkFunction[String] {
+
+  override def invoke(value: String): Unit = {
+    synchronized {
+      certificateIssuedEvents.values.add(value)
+    }
+  }
+}
+
+object certificateIssuedEvents {
+  val values: util.List[String] = new util.ArrayList()
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/content-cache-updater/pom.xml b/data-pipeline-flink/content-cache-updater/pom.xml
index d9b8cdc01d..6d4a37d338 100644
--- a/data-pipeline-flink/content-cache-updater/pom.xml
+++ b/data-pipeline-flink/content-cache-updater/pom.xml
@@ -31,7 +31,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -55,13 +55,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -74,7 +74,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -114,7 +114,14 @@
     		<sourceDirectory>src/main/scala</sourceDirectory>
 		<testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
-
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -163,10 +170,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf b/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf
index 7af48f484f..8a4f7e2bf6 100644
--- a/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf
+++ b/data-pipeline-flink/content-cache-updater/src/main/resources/content-cache-updater.conf
@@ -11,6 +11,10 @@ redis-meta {
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6379
+  dialcode.port = 6379
 }
 
 
diff --git a/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties b/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties
deleted file mode 100644
index 05e98e6331..0000000000
--- a/data-pipeline-flink/content-cache-updater/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=contentcache.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala
index 0f2e75a737..3603bd48fe 100644
--- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala
+++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/domain/Event.scala
@@ -2,24 +2,23 @@ package org.sunbird.dp.contentupdater.domain
 
 import java.util
 
-import com.google.gson.internal.LinkedTreeMap
 import org.sunbird.dp.core.domain.Events
 
 class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
 
     private val jobName = "ContentCacheUpdater"
     import scala.collection.JavaConverters._
-    def extractProperties(): Map[String, Any] = {
-        val properties = telemetry.read[LinkedTreeMap[String, Any]]("transactionData.properties")
+    def extractProperties(): Map[String, AnyRef] = {
+        val properties = telemetry.read[util.Map[String, AnyRef]]("transactionData.properties")
         properties.map { propertySet =>
             propertySet.asScala.map {
-                case (key, value) => key -> value.asInstanceOf[LinkedTreeMap[String, Any]].getOrDefault("nv", None)
+                case(key, value) => key -> value.asInstanceOf[util.Map[String, AnyRef]].getOrDefault("nv", None)
             }.toMap
-        }.getOrElse(Map.empty[String, Any])
+        }.getOrElse(Map[String, AnyRef]())
     }
 
 
     def getNodeUniqueId(): String = {
-        telemetry.read[String]("nodeUniqueId").get
+        telemetry.read[String]("nodeUniqueId").orNull
     }
 }
diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala
index 0d610a46d4..973abd07fe 100644
--- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala
+++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/ContentUpdaterFunction.scala
@@ -24,12 +24,12 @@ class ContentUpdaterFunction(config: ContentCacheUpdaterConfig)(implicit val map
     private lazy val gson = new Gson()
 
     override def metricsList(): List[String] = {
-        List(config.contentCacheHit)
+        List(config.contentCacheHit, config.skippedEventCount)
     }
 
     override def open(parameters: Configuration): Unit = {
         super.open(parameters)
-        dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.contentStore, List())
+        dataCache = new DataCache(config, new RedisConnect(config.contentRedisHost, config.contentRedisPort, config), config.contentStore, List())
         dataCache.init()
     }
 
@@ -41,29 +41,35 @@ class ContentUpdaterFunction(config: ContentCacheUpdaterConfig)(implicit val map
 
     override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = {
         val nodeUniqueId = event.getNodeUniqueId()
-        val redisData = dataCache.getWithRetry(nodeUniqueId)
-        val finalProperties = event.extractProperties().filter(property => null != property._2)
+        if (null != nodeUniqueId) {
+            val redisData = dataCache.getWithRetry(nodeUniqueId)
+            val finalProperties = event.extractProperties().filter(property => null != property._2)
 
-        val newProperties = finalProperties.map { case (property, nv) =>
-            if (config.contentDateFields.contains(property))
-                (property, new SimpleDateFormat(config.contentDateFormat).parse(nv.toString).getTime)
-            else if (config.contentListFields.contains(property))
-                (property, nv match {
-                    case _: String => List(nv)
-                    case _: util.ArrayList[String] => nv
-                })
-            else
-                (property, nv)
-        }.filter(map => None != map._2)
-        redisData ++= newProperties.asInstanceOf[Map[String, AnyRef]]
+            val newProperties = finalProperties.map { case (property, nv) =>
+                if (config.contentDateFields.contains(property) && null != nv && nv.asInstanceOf[String].nonEmpty)
+                    (property, new SimpleDateFormat(config.contentDateFormat).parse(nv.toString).getTime)
+                else if (config.contentListFields.contains(property))
+                    (property, nv match {
+                        case _: String => List(nv)
+                        case _: util.ArrayList[String] => nv
+                    })
+                else
+                    (property, nv)
+            }.filter(map => None != map._2)
+            redisData ++= newProperties.asInstanceOf[Map[String, AnyRef]]
 
-        if (redisData.nonEmpty) {
-            dataCache.setWithRetry(event.getNodeUniqueId(), gson.toJson(redisData.asJava))
-            metrics.incCounter(config.contentCacheHit)
-            logger.info(nodeUniqueId + " Updated Successfully")
-        }
+            if (redisData.nonEmpty) {
+                dataCache.setWithRetry(event.getNodeUniqueId(), gson.toJson(redisData.asJava))
+                metrics.incCounter(config.contentCacheHit)
+                logger.info(nodeUniqueId + " Updated Successfully")
+            }
 
-        if (finalProperties.exists(p => config.dialCodeProperties.contains(p._1)))
-            context.output(config.withContentDailCodeEventsTag, event)
+            if (finalProperties.exists(p => config.dialCodeProperties.contains(p._1)))
+                context.output(config.withContentDailCodeEventsTag, event)
+        }
+        else{
+            metrics.incCounter(config.skippedEventCount)
+            logger.info("Skipping as nodeUniqueId retrieved is null from event. Event json might be invalid")
+        }
     }
 }
diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala
index 7341748a85..9f019d4402 100644
--- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala
+++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/functions/DialCodeUpdaterFunction.scala
@@ -32,7 +32,7 @@ class DialCodeUpdaterFunction(config: ContentCacheUpdaterConfig)
 
     override def open(parameters: Configuration): Unit = {
         super.open(parameters)
-        dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config),
+        dataCache = new DataCache(config, new RedisConnect(config.dialcodeRedisHost, config.dialcodeRedisPort, config),
             config.dialcodeStore, config.dialcodeFields)
         dataCache.init()
     }
diff --git a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala
index 191160eb85..d1803889a2 100644
--- a/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala
+++ b/data-pipeline-flink/content-cache-updater/src/main/scala/org/sunbird/dp/contentupdater/task/ContentCacheUpdaterConfig.scala
@@ -17,10 +17,16 @@ class ContentCacheUpdaterConfig(override val config: Config) extends BaseJobConf
   // Kafka Topics Configuration
   val inputTopic: String = config.getString("kafka.input.topic")
 
+  val contentRedisHost: String = config.getString("redis-meta.content.host")
+  val dialcodeRedisHost: String = config.getString("redis-meta.dialcode.host")
+
+  val contentRedisPort: Int = config.getInt("redis-meta.content.port")
+  val dialcodeRedisPort: Int = config.getInt("redis-meta.dialcode.port")
+
   val contentStore: Int = config.getInt("redis-meta.database.contentstore.id")
   val dialcodeStore: Int = config.getInt("redis-meta.database.dialcodestore.id")
 
-  val contentListFields=List("gradeLevel","subject","medium","language")
+  val contentListFields=List("gradeLevel","subject","medium","language", "keywords")
   val contentDateFields=List("lastStatusChangedOn","lastUpdatedOn","createdOn")
   val dialcodeFields = List( "identifier", "channel", "publisher", "batchCode","status","generatedOn","publishedOn","metadata")
 
@@ -35,6 +41,7 @@ class ContentCacheUpdaterConfig(override val config: Config) extends BaseJobConf
   val withDialCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_DIALCODE_EVENTS)
   val withContentDailCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_CONTENT_EVENTS)
 
+  val skippedEventCount = "skipped-event-count"
   val contentCacheHit = "cache-hit-count"
   val dialCodeCacheHit = "dial-codes-cache-hit-count"
   val dialCodeApiHit = "dial-codes-api-hit-count"
diff --git a/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf b/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf
index 58a01f1dad..3a58e0bed8 100644
--- a/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf
+++ b/data-pipeline-flink/content-cache-updater/src/test/resources/test.conf
@@ -26,6 +26,10 @@ redis-meta {
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6340
+  dialcode.port = 6340
 }
 
 
diff --git a/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf b/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf
index c925536f4d..a473371f41 100644
--- a/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf
+++ b/data-pipeline-flink/content-cache-updater/src/test/resources/test2.conf
@@ -29,6 +29,10 @@ redis-meta {
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6340
+  dialcode.port = 6340
 }
 
 telemetry.ignore.period.months = 6
diff --git a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index 03cf5dc9b8..6524ff940e 100644
--- a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -12,9 +12,14 @@ object EventFixture {
 
     val contentData1 = """{"ets":1585729640345,"channel":"012550822176260096119","transactionData":{"properties":{"ownershipType":{"ov":null,"nv":["createdBy"]},"code":{"ov":null,"nv":"org.sunbird.kgcdDt"},"channel":{"ov":null,"nv":"012550822176260096119"},"description":{"ov":null,"nv":"Enter description for TextBook"},"organisation":{"ov":null,"nv":["diksha_ntptest_org"]},"language":{"ov":null,"nv":["English"]},"mimeType":{"ov":null,"nv":"application/vnd.ekstep.content-collection"},"idealScreenSize":{"ov":null,"nv":"normal"},"createdOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"appId":{"ov":null,"nv":"staging.diksha.portal"},"contentDisposition":{"ov":null,"nv":"inline"},"lastUpdatedOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"contentEncoding":{"ov":null,"nv":"gzip"},"contentType":{"ov":null,"nv":"TextBook"},"dialcodeRequired":{"ov":null,"nv":"No"},"creator":{"ov":null,"nv":"suborg_creator_sun 3"},"createdFor":{"ov":null,"nv":["012550822176260096119"]},"lastStatusChangedOn":{"ov":null,"nv":"2020-04-01T08:25:24.896+0000"},"audience":{"ov":null,"nv":["Learner"]},"IL_SYS_NODE_TYPE":{"ov":null,"nv":"DATA_NODE"},"visibility":{"ov":null,"nv":"Default"},"os":{"ov":null,"nv":["All"]},"consumerId":{"ov":null,"nv":"a9cb3a83-a164-4bf0-aa49-b834cebf1c07"},"mediaType":{"ov":null,"nv":"content"},"osId":{"ov":null,"nv":"org.ekstep.quiz.app"},"version":{"ov":null,"nv":2},"versionKey":{"ov":null,"nv":"1585729524896"},"idealScreenDensity":{"ov":null,"nv":"hdpi"},"license":{"ov":null,"nv":"CC BY 4.0"},"framework":{"ov":null,"nv":"ekstep_ncert_k-12"},"createdBy":{"ov":null,"nv":"7a6b150c-08be-4e31-8f69-7fc4b479e61d"},"compatibilityLevel":{"ov":null,"nv":1},"IL_FUNC_OBJECT_TYPE":{"ov":null,"nv":"Content"},"name":{"ov":null,"nv":"BookA"},"IL_UNIQUE_ID":{"ov":null,"nv":"do_2129902962679398401472"},"resourceType":{"ov":null,"nv":"Book"},"status":{"ov":null,"nv":"Draft"}}},"mid":"46e30999-1358-4c93-a03f-eebfdf83eb15","label":"BookA","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-01T08:27:20.345+0000","objectType":"Content","nodeUniqueId":"do_2129902962679398401472","requestId":null,"operationType":"CREATE","nodeGraphId":215201,"graphId":"domain"}"""
     val contentData2 = """{"ets":1586817395203,"channel":"01241974041332940818","transactionData":{"properties":{"ownershipType":{"ov":["createdBy"],"nv":null},"copyright":{"ov":"CBSE","nv":null},"subject":{"ov":["English"],"nv":"Hindi"},"channel":{"ov":"01241974041332940818","nv":null},"downloadUrl":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar","nv":null},"organisation":{"ov":["CBSE"],"nv":null},"language":{"ov":["English"],"nv":null},"mimeType":{"ov":"application/vnd.ekstep.content-collection","nv":null},"variants":{"ov":"{\"online\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838811143_do_312796455301447680117634_13.0_online.ecar\",\"size\":161521.0},\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar\",\"size\":6522150.0}}","nv":null},"appIcon":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312796455301447680117634/artifact/book_1564719079536.thumb.png","nv":null},"gradeLevel":{"ov":["Class 10"],"nv":["Class 5"]},"me_totalRatingsCount":{"ov":2587,"nv":null},"appId":{"ov":"prod.diksha.portal","nv":null},"contentEncoding":{"ov":"gzip","nv":null},"lockKey":{"ov":"efb3aecc-efde-4078-a26f-a6c21d3ff1aa","nv":null},"mimeTypesCount":{"ov":"{\"application/vnd.ekstep.html-archive\":44,\"application/pdf\":72,\"application/vnd.ekstep.content-collection\":178,\"application/vnd.ekstep.ecml-archive\":88,\"video/mp4\":29}","nv":null},"totalCompressedSize":{"ov":336783683,"nv":null},"contentType":{"ov":"TextBook","nv":null},"lastUpdatedBy":{"ov":"edce4f4f-6c82-458a-8b23-e3521859992f","nv":null},"audience":{"ov":["Learner"],"nv":null},"toc_url":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_312796455301447680117634/artifact/do_312796455301447680117634_toc.json","nv":null},"visibility":{"ov":"Default","nv":null},"contentTypesCount":{"ov":"{\"TextBookUnit\":178,\"PracticeQuestionSet\":88,\"ExplanationResource\":53,\"Resource\":1,\"LearningOutcomeDefinition\":22,\"LessonPlan\":47,\"FocusSpot\":22}","nv":null},"consumerId":{"ov":"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01","nv":null},"author":{"ov":"CBSE","nv":null},"mediaType":{"ov":"content","nv":null},"osId":{"ov":"org.ekstep.quiz.app","nv":null},"lastPublishedBy":{"ov":"99606810-7d5c-4f1f-80b0-36c4a0b4415d","nv":null},"version":{"ov":2,"nv":null},"rejectReasons":{"ov":["Others"],"nv":null},"prevState":{"ov":"Review","nv":null},"license":{"ov":"CC BY 4.0","nv":null},"lastPublishedOn":{"ov":"2019-12-20T10:46:44.043+0000","nv":null},"size":{"ov":6522150,"nv":null},"IL_FUNC_OBJECT_TYPE":{"ov":"ContentImage","nv":null},"rejectComment":{"ov":"download TOC","nv":null},"name":{"ov":"First Flight","nv":null},"status":{"ov":"Processing","nv":null},"code":{"ov":"org.sunbird.dYxVdV","nv":null},"prevStatus":{"ov":"Review","nv":null},"description":{"ov":"First Flight","nv":null},"medium":{"ov":["English"],"nv":null},"posterImage":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31281817869664256011879/artifact/book_1564719079536.png","nv":null},"idealScreenSize":{"ov":"normal","nv":null},"createdOn":{"ov":"2019-07-02T11:34:57.539+0000","nv":null},"reservedDialcodes":{"ov":"{\"D6A7V4\":10,\"F6Z8C1\":14,\"U8H8A6\":18,\"G8B5K7\":21,\"H4N9W3\":20,\"V3A2B6\":4,\"G3V4R7\":1,\"T2K1U9\":3,\"X8F3L3\":24,\"N7B3E4\":8,\"E5I2Q4\":19,\"Y2F9H5\":5,\"K9Q6I1\":22,\"S7S7T9\":17,\"S3C6Z9\":23,\"L6L2X7\":6,\"Z1N4C5\":7,\"Q4Z4Y6\":15,\"S5P5F3\":16,\"G1S2D1\":0,\"R1C9A2\":13,\"B5K7E7\":9,\"L8Y1D1\":2,\"N4A4H9\":11,\"C9X3J8\":12}","nv":null},"copyrightYear":{"ov":2019,"nv":null},"contentDisposition":{"ov":"inline","nv":null},"licenseterms":{"ov":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content.","nv":null},"lastUpdatedOn":{"ov":"2020-04-13T22:34:10.853+0000","nv":null},"SYS_INTERNAL_LAST_UPDATED_ON":{"ov":"2020-04-13T21:51:42.618+0000","nv":null},"dialcodeRequired":{"ov":"Yes","nv":null},"createdFor":{"ov":["01241974041332940818"],"nv":null},"lastStatusChangedOn":{"ov":"2020-04-13T22:34:10.842+0000","nv":null},"os":{"ov":["All"],"nv":null},"IL_SYS_NODE_TYPE":{"ov":"DATA_NODE","nv":null},"pkgVersion":{"ov":13,"nv":null},"versionKey":{"ov":"1586817250853","nv":null},"idealScreenDensity":{"ov":"hdpi","nv":null},"depth":{"ov":0,"nv":null},"s3Key":{"ov":"ecar_files/do_312796455301447680117634/first-flight_1576838806540_do_312796455301447680117634_13.0_spine.ecar","nv":null},"dialcodes":{"ov":["G1S2D1"],"nv":null},"framework":{"ov":"ekstep_ncert_k-12","nv":null},"lastSubmittedOn":{"ov":"2020-04-13T21:51:56.227+0000","nv":null},"me_averageRating":{"ov":4,"nv":null},"createdBy":{"ov":"edce4f4f-6c82-458a-8b23-e3521859992f","nv":null},"leafNodesCount":{"ov":233,"nv":null},"compatibilityLevel":{"ov":1,"nv":null},"IL_UNIQUE_ID":{"ov":"do_312796455301447680117634.img","nv":null},"programId":{"ov":"97691300-7e50-11e9-865c-ad8fa09451f7","nv":null},"board":{"ov":"CBSE","nv":null},"resourceType":{"ov":"Book","nv":null}}},"mid":"070f5dd8-f896-4973-8786-592e0af08dc1","label":"First Flight","nodeType":"DATA_NODE","userId":"edce4f4f-6c82-458a-8b23-e3521859992f","createdOn":"2020-04-13T22:36:35.203+0000","objectType":"ContentImage","nodeUniqueId":"do_312796455301447680117634.img","requestId":null,"operationType":"DELETE","nodeGraphId":1058371,"graphId":"domain"}"""
-    val contentData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"gradeLevel":{"ov":null,"nv":["Class 12"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"EKSTEP"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2020},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-04-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"board":{"ov":null,"nv":"CBSE"},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}"""
+    val contentData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"keywords":{"ov":null,"nv":["Story"]},"gradeLevel":{"ov":null,"nv":["Class 12"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"EKSTEP"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2020},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-04-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"board":{"ov":null,"nv":"CBSE"},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}"""
 
     val contentUpdateData3 = """{"ets":1586888770204,"channel":"0123221758376673287017","transactionData":{"properties":{"gradeLevel":{"ov":null,"nv":["Class 8"]},"appIcon":{"ov":null,"nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31293113605096243214/artifact/0_h2y5p6_1578507818966.png"},"copyright":{"ov":null,"nv":"Ekstep"},"subject":{"ov":null,"nv":["Home Science"]},"copyrightYear":{"ov":null,"nv":2021},"topic":{"ov":null,"nv":[]},"licenseterms":{"ov":null,"nv":"By creating any type of content (resources, books, courses etc.) on DIKSHA, you consent to publish it under the Creative Commons License Framework. Please choose the applicable creative commons license you wish to apply to your content."},"lastUpdatedOn":{"ov":"2020-04-14T18:25:58.919+0000","nv":"2020-06-14T18:26:10.133+0000"},"medium":{"ov":null,"nv":["Sanskrit"]},"versionKey":{"ov":"1586888758919","nv":"1586888770133"}}},"mid":"09be8943-0b8b-4c49-91f9-63fff4ebe58a","label":"ResourceACydney","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2020-04-14T18:26:10.204+0000","objectType":"Content","nodeUniqueId":"do_312999792564027392148","requestId":null,"operationType":"UPDATE","nodeGraphId":1071617,"graphId":"domain"}"""
 
+    val invalidEvent1 = """Type":"DialCode"}"""
+    val invalidEvent2 = """reatedOn":"2021-02-11T07:41:59.691+0000","objectType":"DialCode"}"""
+
+    // lastStatusChangedOn nv value = ""
+    val invalidNewValueEvent = """{"ets":1619804548117,"channel":"0123207707019919361056","transactionData":{"properties":{"lastStatusChangedOn":{"ov":"2021-04-28T22:18:18.898+0000","nv":""},"toc_url":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/content/do_31269950653767680013604/artifact/do_31269950653767680013604_toc.json","nv":""},"contentTypesCount":{"ov":"{\"CourseUnit\":10,\"Resource\":8}","nv":""},"prevStatus":{"ov":"Review","nv":""},"childNodes":{"ov":["do_31269950821131059213610","do_31269950821131878413614","do_31269950821130240013605","do_31269950821131878413613","do_31269950821131878413612","do_31269950821131878413611","do_31269949561247334417064","do_31269948474744832017023","do_31269949442614886417054","do_31269948793305497617032","do_31269949714008473617067","do_31269950821131059213608","do_31269948948928102417036","do_31269950821130240013606","do_31269949803231641613567","do_31269950821130240013607","do_31269948673168179217026","do_31269950821131059213609"],"nv":[]},"downloadUrl":{"ov":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299781_do_31269950653767680013604_2.0_spine.ecar","nv":"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1550233883224_do_31269950653767680013604_1.0_spine.ecar"},"variants":{"ov":"{\"online\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299980_do_31269950653767680013604_2.0_online.ecar\",\"size\":10297.0},\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1619648299781_do_31269950653767680013604_2.0_spine.ecar\",\"size\":85954.0}}","nv":"{\"spine\":{\"ecarUrl\":\"https://ntpproductionall.blob.core.windows.net/ntp-content-production/ecar_files/do_31269950653767680013604/coach-role-clarity_1550233883224_do_31269950653767680013604_1.0_spine.ecar\",\"size\":80030.0}}"},"lastPublishedBy":{"ov":"Diksha Implementation","nv":"08c04736-aa38-4b38-8ad9-8ea2857d38a0"},"pkgVersion":{"ov":2.0,"nv":1.0},"lastPublishedOn":{"ov":"2021-04-28T22:18:19.697+0000","nv":"2019-02-15T12:31:23.167+0000"},"size":{"ov":85954.0,"nv":80030.0},"lastUpdatedOn":{"ov":"2021-04-28T22:18:18.910+0000","nv":"2019-02-15T12:31:20.417+0000"},"SYS_INTERNAL_LAST_UPDATED_ON":{"ov":"2021-04-28T22:18:20.113+0000","nv":""},"totalCompressedSize":{"ov":3.2565897E7,"nv":0},"mimeTypesCount":{"ov":"{\"application/vnd.ekstep.h5p-archive\":8,\"application/vnd.ekstep.content-collection\":10}","nv":""}}},"mid":"3cadb9b9-1199-41e3-b069-76f9bf439b8a","label":"Coach- Role Clarity","nodeType":"DATA_NODE","userId":"ANONYMOUS","createdOn":"2021-04-30T17:42:28.117+0000","objectType":"Collection","nodeUniqueId":"do_31269950653767680013604","requestId":null,"operationType":"UPDATE","nodeGraphId":758267,"graphId":"domain"}"""
 
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala
index a2f9f2a558..a6970c2579 100644
--- a/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala
+++ b/data-pipeline-flink/content-cache-updater/src/test/scala/org/sunbird/dp/spec/ContentUpdaterStreamTaskTest.scala
@@ -19,6 +19,7 @@ import org.sunbird.dp.contentupdater.domain.Event
 import org.sunbird.dp.contentupdater.task.{ContentCacheUpdaterConfig, ContentCacheUpdaterStreamTask}
 import org.sunbird.dp.core.cache.RedisConnect
 import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.core.util.JSONUtil
 import org.sunbird.dp.fixture.EventFixture
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
 import redis.embedded.RedisServer
@@ -92,8 +93,9 @@ class ContentUpdaterStreamTaskTest extends BaseTestSpec {
         when(mockKafkaUtil.kafkaEventSource[Event](contentConfig.inputTopic)).thenReturn(new ContentDialCodeSource)
         val task = new ContentCacheUpdaterStreamTask(contentConfig, mockKafkaUtil)
         task.process()
+        BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.skippedEventCount}").getValue() should be(1)
         BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeApiHit}").getValue() should be(1)
-        BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.contentCacheHit}").getValue() should be(10)
+        BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.contentCacheHit}").getValue() should be(11)
         BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeApiMissHit}").getValue() should be(1)
         BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.dialCodeCacheHit}").getValue() should be(2)
         BaseMetricsReporter.gaugeMetrics(s"${contentConfig.jobName}.${contentConfig.totaldialCodeCount}").getValue() should be(3)
@@ -106,6 +108,7 @@ class ContentUpdaterStreamTaskTest extends BaseTestSpec {
         assert(!contentJedis.get("do_312999792564027392148").contains("Class 12"))
         assert(contentJedis.get("do_312999792564027392148").contains("\"copyright\":\"Ekstep\""))
         assert(!contentJedis.get("do_312999792564027392148").contains("\"copyright\":\"EKSTEP\""))
+        assert(contentJedis.get("do_312999792564027392148").contains(""""keywords":["Story"]"""))
     }
 }
 
@@ -114,16 +117,18 @@ class ContentDialCodeSource extends SourceFunction[Event] {
 
     override def run(ctx: SourceContext[Event]) {
         val gson = new Gson()
-        val event1 = gson.fromJson(EventFixture.contentData1, new util.LinkedHashMap[String, Any]().getClass)
-        val event2 = gson.fromJson(EventFixture.contentData2, new util.LinkedHashMap[String, Any]().getClass)
-        val event3 = gson.fromJson(EventFixture.contentData3, new util.LinkedHashMap[String, Any]().getClass)
-        val event4 = gson.fromJson(EventFixture.dialcodedata1, new util.LinkedHashMap[String, Any]().getClass)
-        val event5 = gson.fromJson(EventFixture.invalid_dialcocedata, new util.LinkedHashMap[String, Any]().getClass)
-        val event6 = gson.fromJson(EventFixture.reserved_dialcocedata, new util.LinkedHashMap[String, Any]().getClass)
-        val event7 = gson.fromJson(EventFixture.dialcodedata2, new util.LinkedHashMap[String, Any]().getClass)
-        val event8 = gson.fromJson(EventFixture.invalid_data, new util.LinkedHashMap[String, Any]().getClass)
-        val event9 = gson.fromJson(EventFixture.empty_dialcode, new util.LinkedHashMap[String, Any]().getClass)
-        val event10 = gson.fromJson(EventFixture.contentUpdateData3, new util.LinkedHashMap[String, Any]().getClass)
+        val event1 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData1)
+        val event2 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData2)
+        val event3 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentData3)
+        val event4 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.dialcodedata1)
+        val event5 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalid_dialcocedata)
+        val event6 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.reserved_dialcocedata)
+        val event7 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.dialcodedata2)
+        val event8 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalid_data)
+        val event9 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.empty_dialcode)
+        val event10 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.contentUpdateData3)
+        val event11 = JSONUtil.deserialize[util.HashMap[String, Any]](EventFixture.invalidNewValueEvent)
+
         ctx.collect(new Event(event1))
         ctx.collect(new Event(event2))
         ctx.collect(new Event(event3))
@@ -134,7 +139,10 @@ class ContentDialCodeSource extends SourceFunction[Event] {
         ctx.collect(new Event(event8))
         ctx.collect(new Event(event9))
         ctx.collect(new Event(event10))
-
+        ctx.collect(new Event(event11))
+        // for invalid event check - EventSerializationSchema returns empty map for invalid JSON.
+        // EX: """Type":"DialCode"}""" and """reatedOn":"2021-02-11T07:41:59.691+0000","objectType":"DialCode"}"""
+        ctx.collect(new Event(new util.HashMap[String, Any]()))
     }
 
     override def cancel() = {
diff --git a/data-pipeline-flink/de-normalization/pom.xml b/data-pipeline-flink/de-normalization/pom.xml
index 278e90d3b3..5c61f6c1df 100644
--- a/data-pipeline-flink/de-normalization/pom.xml
+++ b/data-pipeline-flink/de-normalization/pom.xml
@@ -31,7 +31,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -50,13 +50,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -69,7 +69,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -97,6 +97,14 @@
     		<sourceDirectory>src/main/scala</sourceDirectory>
 		    <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -142,10 +150,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf b/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf
index e00174b253..8e962ffa24 100644
--- a/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf
+++ b/data-pipeline-flink/de-normalization/src/main/resources/de-normalization.conf
@@ -3,21 +3,25 @@ include "base-config.conf"
 kafka {
   input.telemetry.topic = ${job.env}".telemetry.unique"
   input.summary.topic = ${job.env}".telemetry.derived"
-  output.success.topic = ${job.env}".telemetry.denorm"
-  output.summary.topic = ${job.env}".telemetry.derived.unique"
+  telemetry.denorm.output.topic = ${job.env}".telemetry.denorm"
+  summary.denorm.output.topic = ${job.env}".druid.events.summary"
+  summary.unique.events.topic = ${job.env}".telemetry.derived.unique"
   output.failed.topic = ${job.env}".telemetry.failed"
   output.duplicate.topic = ${job.env}".telemetry.duplicate"
   groupId = ${job.env}"-telemetry-denorm-group"
 }
 
 task {
+  window.count = 30
+  window.shards = 1400
   consumer.parallelism = 1
-  denorm.parallelism = 1
-  denorm.sink.parallelism = 1
-  summary.sink.parallelism = 1
-  denorm.summary-dedup.parallelism = 1
+  telemetry.downstream.operators.parallelism = 1
+  summary.downstream.operators.parallelism = 1
 }
 
+skip.events = ["INTERRUPT"]
+permit.eid=["AUDIT"]
+
 redis {
   database {
     duplicationstore.id = 9
@@ -29,8 +33,16 @@ redis {
 redis-meta {
   database {
     devicestore.id = 2
-    userstore.id = 4
+    userstore.id = 12
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  device.host = "localhost"
+  user.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6380
+  device.port = 6381
+  user.port = 6382
+  dialcode.port = 6383
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala
index 98ce77fcb4..8afd313ad3 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Event.scala
@@ -5,9 +5,11 @@ import java.util
 import org.joda.time.{DateTime, DateTimeZone}
 import org.joda.time.format.{DateTimeFormat, DateTimeFormatter}
 import org.sunbird.dp.core.domain.{Events, EventsPath}
+import org.sunbird.dp.denorm.task.DenormalizationConfig
 
 import scala.collection.JavaConverters._
 import scala.collection.mutable.Map
+import scala.collection.mutable
 
 class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
 
@@ -93,17 +95,22 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
     telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L1).orNull
   }
 
-  def objectRollUpl1FieldsPresent(): Boolean = {
+  def objectRollUpl2ID(): String = {
+    telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L2).orNull
+  }
+
+  def objectRollUpFieldsPresent(path: String): Boolean = {
 
-    val objectrollUpl1 = telemetry.read[String](keyPath = EventsPath.OBJECT_ROLLUP_L1).orNull
+    val objectrollUpl1 = telemetry.read[String](keyPath = path).orNull
     null != objectrollUpl1 && !objectrollUpl1.isEmpty
   }
 
-  def checkObjectIdNotEqualsRollUpl1Id(): Boolean = {
-    objectRollUpl1FieldsPresent() && !objectID().equals(objectRollUpl1ID())
+  def checkObjectIdNotEqualsRollUpId(path: String): Boolean = {
+    objectRollUpFieldsPresent(path) && !objectID().equals(objectRollUpl1ID())
   }
 
-  def addUserData(newData: Map[String, AnyRef]) {
+  // def addUserData(newData: Map[String, String]) {
+  def addUserData(newData: mutable.Map[String, AnyRef]) {
     val userdata: util.Map[String, AnyRef] = telemetry.read(EventsPath.USERDATA_PATH).getOrElse(new util.HashMap[String, AnyRef]())
     userdata.putAll(newData.asJava)
     telemetry.add(EventsPath.USERDATA_PATH, userdata)
@@ -129,6 +136,14 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
     setFlag("coll_denorm", true)
   }
 
+  def addL2Data(newData: Map[String, AnyRef]) {
+    val l2Map = new util.HashMap[String, AnyRef]()
+    l2Map.putAll(newData.asJava)
+    telemetry.add(EventsPath.L2_DATA_PATH, l2Map)
+    setFlag("l2_denorm", true)
+  }
+
+
   def getEpochConvertedContentDataMap(data: Map[String, AnyRef]): Map[String, AnyRef] = {
 
     val lastSubmittedOn = data.get("lastsubmittedon")
@@ -198,4 +213,8 @@ class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
     } else ""
   }
 
+  def isValidEventForContentDenorm(config: DenormalizationConfig, objectId: String, objectType: String, eid: String): Boolean = {
+    (null != objectType && (config.permitEid.contains(eid) || !List("user", "qr", "dialcode").contains(objectType.toLowerCase())) && null != objectId)
+  }
+
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala
index e6dca31e98..68958914c0 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/domain/Models.scala
@@ -10,16 +10,24 @@ object Models {
 
 class DeviceProfile(val countryCode: String, val country: String, val stateCode: String, val state: String, val city: String, 
     val districtCustom: String, val stateCodeCustom: String, val stateCustomName: String, val userDeclaredState: String, 
-    val userDeclaredDistrict: String, val devicespec: util.Map[String, String], val firstaccess: Long) {
+    val userDeclaredDistrict: String, val devicespec: util.Map[String, String], val firstaccess: java.lang.Long) {
   
 }
     
  object DeviceProfile {
    
-  def apply(map: scala.collection.mutable.Map[String, String]) = new DeviceProfile(
-      map.getOrElse("country_code", ""), map.getOrElse("country", ""), map.getOrElse("state_code", ""),
-      map.getOrElse("state", ""), map.getOrElse("city", ""), map.getOrElse("district_custom", ""),
-      map.getOrElse("state_code_custom", ""), map.getOrElse("state_custom", ""), map.getOrElse("user_declared_state", ""),
-      map.getOrElse("user_declared_district", ""), Models.gson.fromJson(map.getOrElse("devicespec", "{}"), new util.HashMap[String, AnyRef]().getClass), map.getOrElse("firstaccess", "0").toLong)
-  
+  // def apply(map: scala.collection.mutable.Map[String, String]) = new DeviceProfile(
+  def apply(map: scala.collection.mutable.Map[String, AnyRef]) = new DeviceProfile(
+      map.getOrElse("country_code", "").asInstanceOf[String],
+      map.getOrElse("country", "").asInstanceOf[String],
+      map.getOrElse("state_code", "").asInstanceOf[String],
+      map.getOrElse("state", "").asInstanceOf[String],
+      map.getOrElse("city", "").asInstanceOf[String],
+      map.getOrElse("district_custom", "").asInstanceOf[String],
+      map.getOrElse("state_code_custom", "").asInstanceOf[String],
+      map.getOrElse("state_custom", "").asInstanceOf[String], map.getOrElse("user_declared_state", "").asInstanceOf[String],
+      map.getOrElse("user_declared_district", "").asInstanceOf[String],
+      // Models.gson.fromJson(map.getOrElse("devicespec", "{}").asInstanceOf[String], new util.HashMap[String, AnyRef]().getClass),
+      map.getOrElse("devicespec", new util.HashMap[String, AnyRef]()).asInstanceOf[util.Map[String, String]],
+      java.lang.Long.valueOf(map.getOrElse("firstaccess", "0").asInstanceOf[String]))
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala
index dba0dbbdb2..836e3cc30b 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationFunction.scala
@@ -4,10 +4,12 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.slf4j.LoggerFactory
+import org.sunbird.dp.core.cache.RedisConnect
 import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
 import org.sunbird.dp.denorm.`type`._
 import org.sunbird.dp.denorm.domain.Event
 import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.DenormCache
 
 class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTypeInfo: TypeInformation[Event])
   extends BaseProcessFunction[Event, Event](config) {
@@ -19,17 +21,20 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp
   private[this] var dialcodeDenormalization: DialcodeDenormalization = _
   private[this] var contentDenormalization: ContentDenormalization = _
   private[this] var locationDenormalization: LocationDenormalization = _
+  private[this] var denormCache: DenormCache = _
 
 
   override def metricsList(): List[String] = {
     List(config.eventsExpired, config.userTotal, config.userCacheHit, config.userCacheMiss,
       config.contentTotal, config.contentCacheHit, config.contentCacheMiss, config.deviceTotal,
-      config.deviceCacheHit, config.deviceCacheMiss, config.eventsExpired, config.dialcodeTotal,
-      config.dialcodeCacheHit, config.dialcodeCacheMiss, config.locTotal, config.locCacheHit, config.locCacheMiss)
+      config.deviceCacheHit, config.deviceCacheMiss, config.dialcodeTotal,
+      config.dialcodeCacheHit, config.dialcodeCacheMiss,
+      config.locTotal, config.locCacheHit, config.locCacheMiss, config.eventsSkipped)
   }
 
   override def open(parameters: Configuration): Unit = {
     super.open(parameters)
+    denormCache = new DenormCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config))
     deviceDenormalization = new DeviceDenormalization(config)
     userDenormalization = new UserDenormalization(config)
     dialcodeDenormalization = new DialcodeDenormalization(config)
@@ -39,10 +44,7 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp
 
   override def close(): Unit = {
     super.close()
-    deviceDenormalization.closeDataCache()
-    userDenormalization.closeDataCache()
-    dialcodeDenormalization.closeDataCache()
-    contentDenormalization.closeDataCache()
+    denormCache.close()
   }
 
   override def processElement(event: Event,
@@ -51,13 +53,17 @@ class DenormalizationFunction(config: DenormalizationConfig)(implicit val mapTyp
     if (event.isOlder(config.ignorePeriodInMonths)) { // Skip events older than configured value (default: 3 months)
       metrics.incCounter(config.eventsExpired)
     } else {
-      if ("ME_WORKFLOW_SUMMARY" == event.eid() || !event.eid().contains("SUMMARY")) {
-        val deviceDenormEvent = deviceDenormalization.denormalize(event, metrics)
-        val userDenormEvent = userDenormalization.denormalize(deviceDenormEvent, metrics)
-        val dialcodeDenormEvent = dialcodeDenormalization.denormalize(userDenormEvent, metrics)
-        val contentDenormEvent = contentDenormalization.denormalize(dialcodeDenormEvent, metrics)
-        val locationDenormEvent = locationDenormalization.denormalize(contentDenormEvent, metrics)
-        context.output(config.denormEventsTag, locationDenormEvent)
+      if ("ME_WORKFLOW_SUMMARY" == event.eid() || !(event.eid().contains("SUMMARY") || config.eventsToskip.contains(event.eid()))) {
+        val cacheData = denormCache.getDenormData(event)
+        deviceDenormalization.denormalize(event, cacheData, metrics)
+        userDenormalization.denormalize(event, cacheData, metrics)
+        dialcodeDenormalization.denormalize(event, cacheData, metrics)
+        contentDenormalization.denormalize(event, cacheData, metrics)
+        locationDenormalization.denormalize(event, metrics)
+        context.output(config.denormEventsTag, event)
+      }
+      else {
+        metrics.incCounter(config.eventsSkipped)
       }
     }
   }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala
new file mode 100644
index 0000000000..b6de1d879c
--- /dev/null
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/functions/DenormalizationWindowFunction.scala
@@ -0,0 +1,163 @@
+package org.sunbird.dp.denorm.functions
+
+import java.lang
+
+import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.{GlobalWindow, TimeWindow}
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.core.domain.EventsPath
+import org.sunbird.dp.core.job.{Metrics, WindowBaseProcessFunction}
+import org.sunbird.dp.denorm.`type`._
+import org.sunbird.dp.denorm.domain.Event
+import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.{CacheResponseData, DenormCache, DenormWindowCache}
+import org.apache.flink.util.Collector
+
+import scala.collection.mutable.{Map => MMap}
+import scala.collection.JavaConverters._
+
+case class EventCache(event: Event, contentId: Option[String], collectionId: Option[String], l2Id: Option[String], deviceId: Option[String], userId: Option[String], dialcodeId: Option[String])
+
+case class EventsMetadata(contentMap: MMap[String, AnyRef], collectionMap: MMap[String, AnyRef], l2RollupMap: MMap[String, AnyRef],
+                          deviceMap: MMap[String, AnyRef], userMap: MMap[String, AnyRef], dialcodeMap: MMap[String, AnyRef])
+
+class DenormalizationWindowFunction(config: DenormalizationConfig)(implicit val eventTypeInfo: TypeInformation[Event])
+  extends WindowBaseProcessFunction[Event, Event, Int](config) {
+
+    private[this] val logger = LoggerFactory.getLogger(classOf[DenormalizationWindowFunction])
+
+    private[this] var deviceDenormalization: DeviceDenormalization = _
+    private[this] var userDenormalization: UserDenormalization = _
+    private[this] var dialcodeDenormalization: DialcodeDenormalization = _
+    private[this] var contentDenormalization: ContentDenormalization = _
+    private[this] var locationDenormalization: LocationDenormalization = _
+    private[this] var denormCache: DenormWindowCache = _
+
+    override def metricsList(): List[String] = {
+        List(config.eventsExpired, config.userTotal, config.userCacheHit, config.userCacheMiss,
+            config.contentTotal, config.contentCacheHit, config.contentCacheMiss, config.deviceTotal,
+            config.deviceCacheHit, config.deviceCacheMiss, config.dialcodeTotal,
+            config.dialcodeCacheHit, config.dialcodeCacheMiss,
+            config.locTotal, config.locCacheHit, config.locCacheMiss, config.eventsSkipped)
+    }
+
+    override def open(parameters: Configuration): Unit = {
+        super.open(parameters)
+        denormCache = new DenormWindowCache(config,
+            new RedisConnect(config.contentRedisHost, config.contentRedisPort, config),
+            new RedisConnect(config.deviceRedisHost, config.deviceRedisPort, config),
+            new RedisConnect(config.userRedisHost, config.userRedisPort, config),
+            new RedisConnect(config.dialcodeRedisHost, config.dialcodeRedisPort, config)
+        )
+        deviceDenormalization = new DeviceDenormalization(config)
+        userDenormalization = new UserDenormalization(config)
+        dialcodeDenormalization = new DialcodeDenormalization(config)
+        contentDenormalization = new ContentDenormalization(config)
+        locationDenormalization = new LocationDenormalization(config)
+    }
+
+    override def close(): Unit = {
+        super.close()
+        denormCache.close()
+    }
+
+    override def process(key: Int, context: ProcessWindowFunction[Event, Event, Int, GlobalWindow]#Context, elements: lang.Iterable[Event], metrics: Metrics): Unit = {
+
+        val summaryEventsList = List("ME_WORKFLOW_SUMMARY", "SUMMARY")
+        val eventsList = elements.asScala.toList
+        val filteredEventsList: List[Event] = eventsList.filter { event =>
+            if (event.isOlder(config.ignorePeriodInMonths)) { // Skip events older than configured value (default: 3 months)
+                logger.info(s"Event Dropped: Event older than configured value (default: 3 months)")
+                metrics.incCounter(config.eventsExpired)
+                false
+            } else {
+                if (summaryEventsList.contains(event.eid()) || !(event.eid().contains("SUMMARY") || config.eventsToskip.contains(event.eid()))) {
+                    true
+                } else {
+                    metrics.incCounter(config.eventsSkipped)
+                    false
+                }
+            }
+        }
+        denormalize(filteredEventsList, context, metrics)
+    }
+
+    def denormalize(events: List[Event], context: ProcessWindowFunction[Event, Event, Int, GlobalWindow]#Context, metrics: Metrics) = {
+
+        val eventCacheData = parseLookupIds(events)
+        val eventsDenormMeta = denormCache.getDenormData(eventCacheData._2)
+
+        eventCacheData._1.foreach {
+            eventData =>
+                val event = eventData.event
+                val cacheData = CacheResponseData(
+                    content = eventsDenormMeta.contentMap.getOrElse(eventData.contentId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]],
+                    collection = eventsDenormMeta.collectionMap.getOrElse(eventData.collectionId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]],
+                    l2data = eventsDenormMeta.l2RollupMap.getOrElse(eventData.l2Id.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]],
+                    device = eventsDenormMeta.deviceMap.getOrElse(eventData.deviceId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]],
+                    user = eventsDenormMeta.userMap.getOrElse(eventData.userId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]],
+                    dialCode = eventsDenormMeta.dialcodeMap.getOrElse(eventData.dialcodeId.getOrElse(""), MMap[String, AnyRef]()).asInstanceOf[MMap[String, AnyRef]]
+                )
+                deviceDenormalization.denormalize(event, cacheData, metrics)
+                userDenormalization.denormalize(event, cacheData, metrics)
+                dialcodeDenormalization.denormalize(event, cacheData, metrics)
+                contentDenormalization.denormalize(event, cacheData, metrics)
+                locationDenormalization.denormalize(event, metrics)
+                context.output(config.denormEventsTag, event)
+        }
+    }
+
+    def parseLookupIds(events: List[Event]): (List[EventCache], EventsMetadata) = {
+        val contentMap = MMap[String, AnyRef]()
+        val collectionMap = MMap[String, AnyRef]()
+        val l2RollupMap = MMap[String, AnyRef]()
+        val dialcodeMap = MMap[String, AnyRef]()
+        val deviceMap = MMap[String, AnyRef]()
+        val userMap = MMap[String, AnyRef]()
+
+        val eventsCache = events.map { event =>
+            val objectType = event.objectType()
+            val objectId = event.objectID()
+            val deviceId = if (null != event.did()) Some(event.did()) else None
+            val actorId = event.actorId()
+            val actorType = event.actorType()
+
+            val contentIds = if (event.isValidEventForContentDenorm(config, objectId, objectType, event.eid())) {
+                contentMap.put(objectId, null)
+                val collectionId = if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) {
+                    collectionMap.put(event.objectRollUpl1ID(), null)
+                    Some(event.objectRollUpl1ID())
+                } else None
+
+                val l2RollupId = if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) {
+                    l2RollupMap.put(event.objectRollUpl2ID(), null)
+                    Some(event.objectRollUpl2ID())
+                } else None
+
+                Some((objectId, collectionId, l2RollupId))
+            } else {
+                None
+            }
+
+            val dialcodeId = if (null != objectType && List("dialcode", "qr").contains(objectType.toLowerCase())) {
+                dialcodeMap.put(objectId.toUpperCase(), null)
+                Some(objectId.toUpperCase)
+            } else None
+
+            deviceId.map(did => deviceMap.put(did, null))
+
+            val userId = if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) {
+                userMap.put(config.userStoreKeyPrefix + actorId, null)
+                Some(config.userStoreKeyPrefix + actorId)
+            } else None
+
+            EventCache(event, contentId = contentIds.map(_._1), collectionId = contentIds.flatMap(_._2), l2Id = contentIds.flatMap(_._3), deviceId = deviceId, userId = userId, dialcodeId = dialcodeId)
+        }
+        val eventsMetaCache = EventsMetadata(contentMap, collectionMap, l2RollupMap, deviceMap, userMap, dialcodeMap)
+        (eventsCache, eventsMetaCache)
+    }
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala
index 3c1b521e3c..bd3c275968 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationConfig.scala
@@ -6,8 +6,7 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.streaming.api.scala.OutputTag
 import org.sunbird.dp.core.job.BaseJobConfig
 import org.sunbird.dp.denorm.domain.Event
-
-import scala.collection.JavaConversions._
+import scala.collection.JavaConverters._
 
 class DenormalizationConfig(override val config: Config, jobName: String) extends BaseJobConfig(config, jobName ) {
 
@@ -19,29 +18,44 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend
   // Kafka Topics Configuration
   val telemetryInputTopic: String = config.getString("kafka.input.telemetry.topic")
   val summaryInputTopic: String = config.getString("kafka.input.summary.topic")
-  val denormSuccessTopic: String = config.getString("kafka.output.success.topic")
+  val telemetryDenormOutputTopic: String = config.getString("kafka.telemetry.denorm.output.topic")
+  val summaryDenormOutputTopic: String = config.getString("kafka.summary.denorm.output.topic")
   val failedTopic: String = config.getString("kafka.output.failed.topic")
-  val summaryOutputEventsTopic: String = config.getString("kafka.output.summary.topic")
+  val summaryUniqueEventsTopic: String = config.getString("kafka.summary.unique.events.topic")
 
   override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
-  val denormParallelism: Int = config.getInt("task.denorm.parallelism")
-  val denormSinkParallelism: Int = config.getInt("task.denorm.sink.parallelism")
-  val summarySinkParallelism: Int = config.getInt("task.summary.sink.parallelism")
+  val telemetryDownstreamOperatorsParallelism: Int = config.getInt("task.telemetry.downstream.operators.parallelism")
+  val summaryDownstreamOperatorsParallelism: Int = config.getInt("task.summary.downstream.operators.parallelism")
+
+  // Windows
+  val windowCount: Int = config.getInt("task.window.count")
+  val windowShards: Int = config.getInt("task.window.shards")
 
   val userStore: Int = config.getInt("redis-meta.database.userstore.id")
   val contentStore: Int = config.getInt("redis-meta.database.contentstore.id")
   val deviceStore: Int = config.getInt("redis-meta.database.devicestore.id")
   val dialcodeStore: Int = config.getInt("redis-meta.database.dialcodestore.id")
 
+  val userRedisHost: String = config.getString("redis-meta.user.host")
+  val deviceRedisHost: String = config.getString("redis-meta.device.host")
+  val contentRedisHost: String = config.getString("redis-meta.content.host")
+  val dialcodeRedisHost: String = config.getString("redis-meta.dialcode.host")
+
+  val userRedisPort: Int = config.getInt("redis-meta.user.port")
+  val deviceRedisPort: Int = config.getInt("redis-meta.device.port")
+  val contentRedisPort: Int = config.getInt("redis-meta.content.port")
+  val dialcodeRedisPort: Int = config.getInt("redis-meta.dialcode.port")
+
   val deviceFields = List("country_code", "country", "state_code", "state", "city", "district_custom", "state_code_custom",
     "state_custom", "user_declared_state", "user_declared_district", "devicespec", "firstaccess")
   val contentFields = List("name", "objectType", "contentType", "mediaType", "language", "medium", "mimeType", "createdBy",
-    "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn", "channel")
-  val userFields = List("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype")
+    "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn", "channel", "gradeLevel", "keywords")
+  val l2DataFields = List("name", "contentType", "mimeType", "framework", "subject", "medium", "board", "channel", "createdFor", "gradeLevel")
+  val userFields = List("usertype", "usersubtype","grade", "language", "subject", "state", "district", "usersignintype", "userlogintype", "block", "cluster", "schoolname")
   val dialcodeFields = List("identifier", "channel", "batchcode", "publisher", "generated_on", "published_on", "status")
   
   val ignorePeriodInMonths:Int = if(config.hasPath("telemetry.ignore.period.months")) config.getInt("telemetry.ignore.period.months") else 3
-  val summaryFilterEvents: List[String] = if(config.hasPath("summary.filter.events")) config.getStringList("summary.filter.events").toList else List("ME_WORKFLOW_SUMMARY")
+  val summaryFilterEvents: List[String] = if(config.hasPath("summary.filter.events")) config.getStringList("summary.filter.events").asScala.toList else List("ME_WORKFLOW_SUMMARY")
   
   val userSignInTypeDefault: String = if (config.hasPath("user.signin.type.default")) config.getString("user.signin.type.default") else "Anonymous"
   val userLoginInTypeDefault: String = if (config.hasPath("user.login.type.default")) config.getString("user.login.type.default") else "NA"
@@ -62,6 +76,10 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend
   val withDialCodeEventsTag: OutputTag[Event] = OutputTag[Event](WITH_DIALCODE_EVENTS)
   val denormEventsTag: OutputTag[Event] = OutputTag[Event](DENORM_EVENTS)
 
+  val eventsToskip: List[String] = config.getStringList("skip.events").asScala.toList
+  val permitEid: List[String] = config.getStringList("permit.eid").asScala.toList
+  val eventsSkipped = "events-skipped"
+
   // Device Denorm Metrics
   val deviceTotal = "device-total"
   val deviceCacheHit = "device-cache-hit"
@@ -118,11 +136,10 @@ class DenormalizationConfig(override val config: Config, jobName: String) extend
   val summaryDedupFunction = "SummaryDeduplicationFunction"
   val summaryDenormalizationFunction = "SummaryDenormalizationFunction"
 
-  val summaryDedupParallelism: Int = config.getInt("task.denorm.summary-dedup.parallelism")
-  val summarydenormParallelism: Int = config.getInt("task.denorm.parallelism")
-  val summaryDenormSinkParallelism: Int = config.getInt("task.summary.sink.parallelism")
-
   // Metrics
   val summaryEventsCount = "summary-events-count"
 
+  //user store key prefix
+  val userStoreKeyPrefix = "user:"
+
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala
index 4b434d6cac..8531e8396c 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/DenormalizationStreamTask.scala
@@ -4,6 +4,7 @@ import java.io.File
 
 import com.typesafe.config.ConfigFactory
 import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.functions.KeySelector
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.api.java.utils.ParameterTool
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
@@ -61,12 +62,13 @@ class DenormalizationStreamTask(config: DenormalizationConfig, kafkaConnector: F
     val denormStream =
       env.addSource(source, config.denormalizationConsumer).uid(config.denormalizationConsumer)
         .setParallelism(config.kafkaConsumerParallelism).rebalance()
-        .process(new DenormalizationFunction(config)).name(config.denormalizationFunction).uid(config.denormalizationFunction)
-          .setParallelism(config.denormParallelism)
+        .keyBy(new DenormKeySelector(config)).countWindow(config.windowCount)
+        .process(new DenormalizationWindowFunction(config)).name(config.denormalizationFunction).uid(config.denormalizationFunction)
+        .setParallelism(config.telemetryDownstreamOperatorsParallelism)
 
-    denormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.denormSuccessTopic))
+    denormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.telemetryDenormOutputTopic))
       .name(config.DENORM_EVENTS_PRODUCER).uid(config.DENORM_EVENTS_PRODUCER)
-        .setParallelism(config.denormSinkParallelism)
+        .setParallelism(config.telemetryDownstreamOperatorsParallelism)
 
     env.execute(config.jobName)
   }
@@ -88,3 +90,10 @@ object DenormalizationStreamTask {
   }
 }
 // $COVERAGE-ON$
+
+class DenormKeySelector(config: DenormalizationConfig) extends KeySelector[Event, Int] {
+  val shards = config.windowShards
+  override def getKey(in: Event): Int = {
+    if (Option(in.did()) == None) "".hashCode % shards else in.did().hashCode % shards
+  }
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala
index fdf2e77e11..84aaa41682 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/task/SummaryDenormalizationStreamTask.scala
@@ -8,10 +8,9 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.api.java.utils.ParameterTool
 import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.sunbird.dp.denorm.domain.Event
-import org.sunbird.dp.denorm.functions.DenormalizationFunction
+import org.sunbird.dp.denorm.functions.{DenormalizationFunction, DenormalizationWindowFunction, SummaryDeduplicationFunction}
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.core.util.FlinkUtil
-import org.sunbird.dp.denorm.functions.SummaryDeduplicationFunction
 
 /**
   * Denormalization stream task does the following pipeline processing in a sequence:
@@ -61,25 +60,27 @@ class SummaryDenormalizationStreamTask(config: DenormalizationConfig, kafkaConne
       env.addSource(source, config.summaryDenormalizationConsumer).uid(config.summaryDenormalizationConsumer)
         .setParallelism(config.kafkaConsumerParallelism).rebalance()
         .process(new SummaryDeduplicationFunction(config)).name(config.summaryDedupFunction).uid(config.summaryDedupFunction)
-        .setParallelism(config.summaryDedupParallelism)
+        .setParallelism(config.summaryDownstreamOperatorsParallelism)
 
     val summaryDenormStream = summaryEventStream.getSideOutput(config.uniqueSummaryEventsOutputTag)
-      .process(new DenormalizationFunction(config))
+      .keyBy(new DenormKeySelector(config)).countWindow(config.windowCount)
+      .process(new DenormalizationWindowFunction(config))
       .name(config.summaryDenormalizationFunction).uid(config.summaryDenormalizationFunction)
-      .setParallelism(config.denormParallelism)
+      .setParallelism(config.summaryDownstreamOperatorsParallelism)
 
     summaryEventStream.getSideOutput(config.duplicateEventsOutputTag)
       .addSink(kafkaConnector.kafkaEventSink[Event](config.duplicateTopic))
       .name(config.summaryDuplicateEventProducer).uid(config.summaryDuplicateEventProducer)
+      .setParallelism(config.summaryDownstreamOperatorsParallelism)
 
-    summaryDenormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.denormSuccessTopic))
+    summaryDenormStream.getSideOutput(config.denormEventsTag).addSink(kafkaConnector.kafkaEventSink(config.summaryDenormOutputTopic))
       .name(config.summaryDenormEventsProducer).uid(config.summaryDenormEventsProducer)
-      .setParallelism(config.denormSinkParallelism)
+      .setParallelism(config.summaryDownstreamOperatorsParallelism)
 
     summaryEventStream.getSideOutput(config.uniqueSummaryEventsOutputTag)
-      .addSink(kafkaConnector.kafkaEventSink(config.summaryOutputEventsTopic))
+      .addSink(kafkaConnector.kafkaEventSink(config.summaryUniqueEventsTopic))
       .name(config.summaryEventsProducer).uid(config.summaryEventsProducer)
-      .setParallelism(config.summarySinkParallelism)
+      .setParallelism(config.summaryDownstreamOperatorsParallelism)
 
     env.execute(config.jobName)
   }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala
index 5adfea1602..83cc453961 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/ContentDenormalization.scala
@@ -1,23 +1,21 @@
 package org.sunbird.dp.denorm.`type`
 
-import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
+import org.sunbird.dp.core.domain.EventsPath
 import org.sunbird.dp.core.job.Metrics
 import org.sunbird.dp.denorm.domain.Event
 import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.CacheResponseData
 
 class ContentDenormalization(config: DenormalizationConfig) {
 
-  private val contentDataCache =
-    new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config),
-      config.contentStore, config.contentFields)
-  contentDataCache.init()
-
-  def denormalize(event: Event, metrics: Metrics): Event = {
+  def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = {
     val objectType = event.objectType()
     val objectId = event.objectID()
-    if (!List("user", "qr", "dialcode").contains(objectType) && null != objectId) {
+    if (event.isValidEventForContentDenorm(config, objectId, objectType, event.eid())) {
       metrics.incCounter(config.contentTotal)
-      val contentData = contentDataCache.getWithRetry(objectId).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)})
+      val contentData = cacheData.content.map(f => {
+        (f._1.toLowerCase().replace("_", ""), f._2)
+      })
 
       if (contentData.nonEmpty) {
         metrics.incCounter(config.contentCacheHit)
@@ -27,14 +25,17 @@ class ContentDenormalization(config: DenormalizationConfig) {
         event.setFlag("content_denorm", value = false)
       }
 
-      if (event.checkObjectIdNotEqualsRollUpl1Id()) {
-        event.addCollectionData(contentDataCache.getWithRetry(event.objectRollUpl1ID()))
+      if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) {
+        event.addCollectionData(cacheData.collection.map(f => {
+          (f._1.toLowerCase().replace("_", ""), f._2)
+        }))
+      }
+      if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) {
+        event.addL2Data(cacheData.l2data.filter(x => config.l2DataFields.contains(x._1)).map(f => {
+          (f._1.toLowerCase().replace("_", ""), f._2)
+        }))
       }
     }
-    event
   }
 
-  def closeDataCache() = {
-    contentDataCache.close()
-  }
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala
index 99c5f2f41a..8f35ac4fd1 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DeviceDenormalization.scala
@@ -1,23 +1,18 @@
 package org.sunbird.dp.denorm.`type`
 
-import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
 import org.sunbird.dp.core.job.Metrics
 import org.sunbird.dp.denorm.domain.{DeviceProfile, Event}
 import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.CacheResponseData
 
 class DeviceDenormalization(config: DenormalizationConfig) {
 
-  private val deviceDataCache: DataCache =
-    new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config),
-      config.deviceStore, config.deviceFields)
-  deviceDataCache.init()
-
-  def denormalize(event: Event, metrics: Metrics): Event = {
+  def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = {
     event.compareAndAlterEts() // Reset ets to today's date if we get future value
     val did = event.did()
     if (null != did && did.nonEmpty) {
       metrics.incCounter(config.deviceTotal)
-      val deviceDetails = deviceDataCache.hgetAllWithRetry(did)
+      val deviceDetails = cacheData.device
       if (deviceDetails.nonEmpty) {
         metrics.incCounter(config.deviceCacheHit)
         event.addDeviceProfile(DeviceProfile.apply(deviceDetails))
@@ -26,10 +21,6 @@ class DeviceDenormalization(config: DenormalizationConfig) {
         event.setFlag("device_denorm", value = false)
       }
     }
-    event
   }
 
-  def closeDataCache() = {
-    deviceDataCache.close()
-  }
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala
index 229d603f96..d5bc16215c 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/DialcodeDenormalization.scala
@@ -1,21 +1,16 @@
 package org.sunbird.dp.denorm.`type`
 
-import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
 import org.sunbird.dp.core.job.Metrics
 import org.sunbird.dp.denorm.domain.Event
 import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.CacheResponseData
 
 class DialcodeDenormalization(config: DenormalizationConfig) {
 
-  private val dialcodeDataCache =
-    new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config),
-      config.dialcodeStore, config.dialcodeFields)
-  dialcodeDataCache.init()
-
-  def denormalize(event: Event, metrics: Metrics): Event = {
+  def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = {
     if (null != event.objectType() && List("dialcode", "qr").contains(event.objectType().toLowerCase())) {
       metrics.incCounter(config.dialcodeTotal)
-      val dialcodeData = dialcodeDataCache.getWithRetry(event.objectID().toUpperCase()).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)})
+      val dialcodeData = cacheData.dialCode.map(f => {(f._1.toLowerCase().replace("_", ""), f._2)})
 
       if (dialcodeData.nonEmpty) {
         metrics.incCounter(config.dialcodeCacheHit)
@@ -25,11 +20,6 @@ class DialcodeDenormalization(config: DenormalizationConfig) {
         event.setFlag("dialcode_denorm", value = false)
       }
     }
-    event
-  }
-
-  def closeDataCache() = {
-    dialcodeDataCache.close()
   }
 
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala
index a47e0e53fb..74bef1e9cf 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/LocationDenormalization.scala
@@ -6,14 +6,16 @@ import org.sunbird.dp.denorm.task.DenormalizationConfig
 
 class LocationDenormalization(config: DenormalizationConfig) {
 
-  def denormalize(event: Event, metrics: Metrics): Event = {
+  def denormalize(event: Event, metrics: Metrics) = {
     metrics.incCounter(config.locTotal)
     val userProfileLocation = event.getUserProfileLocation()
     val userDeclaredLocation = event.getUserDeclaredLocation()
     val ipLocation = event.getIpLocation()
 
     val declaredLocation = if (nonEmpty(userProfileLocation)) userProfileLocation
-    else if (nonEmpty(userDeclaredLocation)) userDeclaredLocation else ipLocation
+    else if (nonEmpty(userDeclaredLocation)) {
+      if(userDeclaredLocation.get._1.nonEmpty) userDeclaredLocation else ipLocation
+    } else ipLocation
 
     if (nonEmpty(declaredLocation)) {
       event.addDerivedLocation(declaredLocation.get)
@@ -21,7 +23,6 @@ class LocationDenormalization(config: DenormalizationConfig) {
     } else {
       metrics.incCounter(config.locCacheMiss)
     }
-    event
   }
 
   private def nonEmpty(loc: Option[(String, String, String)]): Boolean = {
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala
index 9a2ee31788..0e202dd058 100644
--- a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/type/UserDenormalization.scala
@@ -1,23 +1,25 @@
 package org.sunbird.dp.denorm.`type`
 
+import scala.collection.mutable.Map
 import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
 import org.sunbird.dp.core.job.Metrics
 import org.sunbird.dp.denorm.domain.Event
 import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.denorm.util.CacheResponseData
 
-class UserDenormalization(config: DenormalizationConfig) {
+import scala.collection.mutable
 
-  private val userDataCache =
-    new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config),
-      config.userStore, config.userFields)
-  userDataCache.init()
+class UserDenormalization(config: DenormalizationConfig) {
 
-  def denormalize(event: Event, metrics: Metrics): Event = {
+  def denormalize(event: Event, cacheData: CacheResponseData, metrics: Metrics) = {
     val actorId = event.actorId()
     val actorType = event.actorType()
-    if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && "user".equalsIgnoreCase(actorType)) {
+    if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) &&
+      ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) {
+
       metrics.incCounter(config.userTotal)
-      val userData = userDataCache.getWithRetry(actorId).map(f => {(f._1.toLowerCase().replace("_", ""), f._2)})
+      val userData: mutable.Map[String, AnyRef] =
+        cacheData.user.map(f => {(f._1.toLowerCase().replace("_", ""), f._2)})
 
       if (userData.isEmpty) {
         metrics.incCounter(config.userCacheMiss)
@@ -25,16 +27,11 @@ class UserDenormalization(config: DenormalizationConfig) {
         metrics.incCounter(config.userCacheHit)
       }
       if (!userData.contains("usersignintype"))
-        userData.put("usersignintype", config.userSignInTypeDefault)
+        userData += "usersignintype" -> config.userSignInTypeDefault
       if (!userData.contains("userlogintype"))
-        userData.put("userlogintype", config.userLoginInTypeDefault)
+        userData += "userlogintype" -> config.userLoginInTypeDefault
       event.addUserData(userData)
     }
-    event
-  }
-
-  def closeDataCache() = {
-    userDataCache.close()
   }
 
 }
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala
new file mode 100644
index 0000000000..0e0261bb07
--- /dev/null
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormBaseCache.scala
@@ -0,0 +1,74 @@
+package org.sunbird.dp.denorm.util
+
+import java.util
+import com.google.gson.Gson
+import org.slf4j.LoggerFactory
+import redis.clients.jedis.Response
+
+import scala.collection.mutable.{Map => MMap}
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+trait DenormBaseCache {
+
+    val gson = new Gson()
+    private[this] val logger = LoggerFactory.getLogger(classOf[DenormBaseCache])
+
+    def getData(data: Response[java.util.Map[String, String]], fields: List[String]): MMap[String, String] = {
+        val dataMap = data.get()
+        if (dataMap.size() > 0) {
+            if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
+            dataMap.values().removeAll(util.Collections.singleton(""))
+            dataMap.asScala
+        } else {
+            MMap[String, String]()
+        }
+    }
+
+    def getDataMap(dataStr: Response[String], fields: List[String]): MMap[String, AnyRef] = {
+        val data = dataStr.get
+        if (data != null && !data.isEmpty) {
+            val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass)
+            if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
+            dataMap.values().removeAll(util.Collections.singleton(""))
+            dataMap.asScala
+        } else {
+            MMap[String, AnyRef]()
+        }
+    }
+
+    def isArray(value: String): Boolean = {
+        val redisValue = value.trim
+        redisValue.length > 0 && redisValue.startsWith("[")
+    }
+
+    def isObject(value: String) = {
+        val redisValue = value.trim
+        redisValue.length > 0 && redisValue.startsWith("{")
+    }
+
+    def convertToComplexDataTypes(data: mutable.Map[String, String]): MMap[String, AnyRef] = {
+        val result = mutable.Map[String, AnyRef]()
+        data.keys.map {
+            redisKey =>
+                val redisValue = data(redisKey)
+                try {
+                    if (isArray(redisValue)) {
+                        result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass)
+                    } else if (isObject(redisValue)) {
+                        result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass)
+                    } else {
+                        result += redisKey -> redisValue.replaceAll("\\\\", "")
+                    }
+                }
+                catch {
+                    case ex: Exception =>
+                        logger.error("Denorm convertToComplexDataTypes failure for redis key : " + redisKey + " and value : " + redisValue)
+                        logger.error("Denorm convertToComplexDataTypes failure :", ex)
+                        throw ex
+                }
+        }
+        result
+    }
+
+}
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala
new file mode 100644
index 0000000000..faac7feec4
--- /dev/null
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormCache.scala
@@ -0,0 +1,160 @@
+package org.sunbird.dp.denorm.util
+
+import java.util
+import com.google.gson.Gson
+import org.slf4j.LoggerFactory
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+import scala.collection.mutable.{Map => MMap}
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.denorm.domain.Event
+import redis.clients.jedis.Pipeline
+import org.sunbird.dp.denorm.task.DenormalizationConfig
+import org.sunbird.dp.core.domain.EventsPath
+import redis.clients.jedis.Response
+
+case class CacheResponseData(content: MMap[String, AnyRef], collection: MMap[String, AnyRef], l2data: MMap[String, AnyRef], device: MMap[String, AnyRef],
+                             dialCode: MMap[String, AnyRef], user: MMap[String, AnyRef])
+
+class DenormCache(val config: DenormalizationConfig, val redisConnect: RedisConnect) {
+
+  private[this] val logger = LoggerFactory.getLogger(classOf[DenormCache])
+  private val pipeline: Pipeline = redisConnect.getConnection(0).pipelined()
+  val gson = new Gson()
+
+  def close() {
+    this.pipeline.close()
+  }
+
+  def getDenormData(event: Event): CacheResponseData = {
+    this.pipeline.clear()
+    val responses = MMap[String, AnyRef]()
+    getContentCache(event, responses)
+    getDeviceCache(event, responses)
+    getDialcodeCache(event, responses)
+    getUserCache(event, responses)
+    this.pipeline.sync()
+    parseResponses(responses)
+  }
+
+  private def getContentCache(event: Event, responses: MMap[String, AnyRef]) {
+    this.pipeline.select(config.contentStore)
+    val objectType = event.objectType()
+    val objectId = event.objectID()
+    if (!List("user", "qr", "dialcode").contains(objectType) && null != objectId) {
+      responses.put("content", this.pipeline.get(objectId).asInstanceOf[AnyRef])
+
+      if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L1)) {
+        responses.put("collection", this.pipeline.get(event.objectRollUpl1ID()).asInstanceOf[AnyRef])
+      }
+      if (event.checkObjectIdNotEqualsRollUpId(EventsPath.OBJECT_ROLLUP_L2)) {
+        responses.put("l2data", this.pipeline.get(event.objectRollUpl2ID()).asInstanceOf[AnyRef])
+      }
+    }
+  }
+
+  private def getDialcodeCache(event: Event, responses: MMap[String, AnyRef]) {
+    this.pipeline.select(config.dialcodeStore)
+    if (null != event.objectType() && List("dialcode", "qr").contains(event.objectType().toLowerCase())) {
+      responses.put("dialcode", this.pipeline.get(event.objectID().toUpperCase()).asInstanceOf[AnyRef])
+    }
+  }
+
+  private def getDeviceCache(event: Event, responses: MMap[String, AnyRef]) {
+    this.pipeline.select(config.deviceStore)
+    if (null != event.did() && event.did().nonEmpty) {
+      responses.put("device", this.pipeline.hgetAll(event.did()).asInstanceOf[AnyRef])
+    }
+  }
+
+  private def getUserCache(event: Event, responses: scala.collection.mutable.Map[String, AnyRef]) {
+    this.pipeline.select(config.userStore)
+    val actorId = event.actorId()
+    val actorType = event.actorType()
+    if (null != actorId && actorId.nonEmpty && !"anonymous".equalsIgnoreCase(actorId) && ("user".equalsIgnoreCase(Option(actorType).getOrElse("")) || "ME_WORKFLOW_SUMMARY".equals(event.eid()))) {
+      responses.put("user", this.pipeline.hgetAll(config.userStoreKeyPrefix + actorId).asInstanceOf[AnyRef])
+    }
+  }
+
+  private def parseResponses(responses: MMap[String, AnyRef]) : CacheResponseData = {
+
+    val userData = responses.get("user").map(data => {
+      convertToComplexDataTypes(getData(data.asInstanceOf[Response[java.util.Map[String, String]]], config.userFields))
+    }).getOrElse(MMap[String, AnyRef]())
+
+    val deviceData = responses.get("device").map(data => {
+      convertToComplexDataTypes(getData(data.asInstanceOf[Response[java.util.Map[String, String]]], config.deviceFields))
+    }).getOrElse(MMap[String, AnyRef]())
+
+    val contentData = responses.get("content").map(data => {
+      getDataMap(data.asInstanceOf[Response[String]], config.contentFields)
+    }).getOrElse(MMap[String, AnyRef]())
+
+    val collectionData = responses.get("collection").map(data => {
+      getDataMap(data.asInstanceOf[Response[String]], config.contentFields)
+    }).getOrElse(MMap[String, AnyRef]())
+
+    val l2Data = responses.get("l2data").map(data => {
+      getDataMap(data.asInstanceOf[Response[String]], config.contentFields)
+    }).getOrElse(MMap[String, AnyRef]())
+
+    val dialData = responses.get("dialcode").map(data => {
+      getDataMap(data.asInstanceOf[Response[String]], config.dialcodeFields)
+    }).getOrElse(MMap[String, AnyRef]())
+
+    CacheResponseData(contentData, collectionData, l2Data, deviceData, dialData, userData)
+  }
+
+  private def getData(data: Response[java.util.Map[String, String]], fields: List[String]): MMap[String, String] = {
+    val dataMap = data.get()
+    if (dataMap.size() > 0) {
+      if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
+      dataMap.values().removeAll(util.Collections.singleton(""))
+      dataMap.asScala
+    } else {
+      MMap[String, String]()
+    }
+  }
+
+  private def getDataMap(dataStr: Response[String], fields: List[String]): MMap[String, AnyRef] = {
+    val data = dataStr.get
+    if (data != null && !data.isEmpty) {
+      val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass)
+      if (fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
+      dataMap.values().removeAll(util.Collections.singleton(""))
+      dataMap.asScala
+    } else {
+      MMap[String, AnyRef]()
+    }
+  }
+
+  def isArray(value: String): Boolean = {
+    val redisValue = value.trim
+    redisValue.length > 0 && redisValue.startsWith("[")
+  }
+
+  def isObject(value: String) = {
+    val redisValue = value.trim
+    redisValue.length > 0 && redisValue.startsWith("{")
+  }
+
+  def convertToComplexDataTypes(data: mutable.Map[String, String]): MMap[String, AnyRef] = {
+    val result = mutable.Map[String, AnyRef]()
+    data.keys.map {
+      redisKey =>
+        val redisValue = data(redisKey)
+        if (isArray(redisValue)) {
+          result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass)
+        } else if (isObject(redisValue)) {
+          result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass)
+        } else {
+          result += redisKey -> redisValue
+        }
+    }
+    result
+  }
+
+}
+
+// $COVERAGE-ON$
diff --git a/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala
new file mode 100644
index 0000000000..4c6ecc5ce2
--- /dev/null
+++ b/data-pipeline-flink/de-normalization/src/main/scala/org/sunbird/dp/denorm/util/DenormWindowCache.scala
@@ -0,0 +1,106 @@
+package org.sunbird.dp.denorm.util
+
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.denorm.functions.EventsMetadata
+import org.sunbird.dp.denorm.task.DenormalizationConfig
+import redis.clients.jedis.Response
+
+import scala.collection.mutable.{Map => MMap}
+
+class DenormWindowCache(config: DenormalizationConfig, contentRedis: RedisConnect, deviceRedis: RedisConnect,
+                        userRedis: RedisConnect, dialcodeRedis: RedisConnect) extends DenormBaseCache {
+
+    private val contentPipeline = contentRedis.getConnection(config.contentStore).pipelined()
+    private val devicePipeline = deviceRedis.getConnection(config.deviceStore).pipelined()
+    private val userPipeline = userRedis.getConnection(config.userStore).pipelined()
+    private val dialcodePipeline = dialcodeRedis.getConnection(config.dialcodeStore).pipelined()
+
+    def close(): Unit = {
+        contentPipeline.close()
+        devicePipeline.close()
+        userPipeline.close()
+        dialcodePipeline.close()
+    }
+
+    def getDenormData(eventsMeta: EventsMetadata): EventsMetadata = {
+        contentPipeline.clear()
+        devicePipeline.clear()
+        userPipeline.clear()
+        dialcodePipeline.clear()
+        getContentCacheData(eventsMeta.contentMap, eventsMeta.collectionMap, eventsMeta.l2RollupMap)
+        getDeviceCacheData(eventsMeta.deviceMap)
+        getUserCacheData(eventsMeta.userMap)
+        getDialcodeCacheData(eventsMeta.dialcodeMap)
+        contentPipeline.sync()
+        devicePipeline.sync()
+        userPipeline.sync()
+        dialcodePipeline.sync()
+        parseResponses(eventsMeta)
+        eventsMeta
+    }
+
+    private def getContentCacheData(contentMap: MMap[String, AnyRef], collectionMap: MMap[String, AnyRef], l2rollupMap: MMap[String, AnyRef]) = {
+        contentMap.keySet.foreach {
+            contentId => contentMap.put(contentId, contentPipeline.get(contentId))
+        }
+
+        collectionMap.keySet.foreach {
+            collectionId => collectionMap.put(collectionId, contentPipeline.get(collectionId))
+        }
+
+        l2rollupMap.keySet.foreach {
+            l2RollupId => l2rollupMap.put(l2RollupId, contentPipeline.get(l2RollupId))
+        }
+    }
+
+    private def getDeviceCacheData(deviceMap: MMap[String, AnyRef]) = {
+        deviceMap.keySet.foreach {
+            deviceId => deviceMap.put(deviceId, devicePipeline.hgetAll(deviceId))
+        }
+    }
+
+    private def getUserCacheData(userMap: MMap[String, AnyRef]) = {
+        userMap.keySet.foreach {
+            userId => userMap.put(userId, userPipeline.hgetAll(userId))
+        }
+    }
+
+    private def getDialcodeCacheData(dialcodeMap: MMap[String, AnyRef]) = {
+        dialcodeMap.keySet.foreach {
+            dialcodeId => dialcodeMap.put(dialcodeId, dialcodePipeline.get(dialcodeId))
+        }
+    }
+
+    def parseResponses(eventsMeta: EventsMetadata) = {
+        eventsMeta.contentMap.foreach {
+            case (contentId, contentResponse) =>
+                eventsMeta.contentMap.put(contentId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields))
+        }
+
+        eventsMeta.collectionMap.foreach {
+            case (collectionId, contentResponse) =>
+                eventsMeta.collectionMap.put(collectionId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields))
+        }
+
+        eventsMeta.l2RollupMap.foreach {
+            case (l2RollupId, contentResponse) =>
+                eventsMeta.l2RollupMap.put(l2RollupId, getDataMap(contentResponse.asInstanceOf[Response[String]], config.contentFields))
+        }
+
+        eventsMeta.deviceMap.foreach {
+            case (deviceId, deviceResponse) =>
+                eventsMeta.deviceMap.put(deviceId, convertToComplexDataTypes(getData(deviceResponse.asInstanceOf[Response[java.util.Map[String, String]]], config.deviceFields)))
+        }
+
+        eventsMeta.userMap.foreach {
+            case (userId, userResponse) =>
+                eventsMeta.userMap.put(userId, convertToComplexDataTypes(getData(userResponse.asInstanceOf[Response[java.util.Map[String, String]]], config.userFields)))
+        }
+
+        eventsMeta.dialcodeMap.foreach {
+            case (dialcodeId, dialcodeResponse) =>
+                eventsMeta.dialcodeMap.put(dialcodeId, getDataMap(dialcodeResponse.asInstanceOf[Response[String]], config.dialcodeFields))
+        }
+
+    }
+}
diff --git a/data-pipeline-flink/de-normalization/src/test/resources/test.conf b/data-pipeline-flink/de-normalization/src/test/resources/test.conf
index 397072f343..d8b87ad70b 100644
--- a/data-pipeline-flink/de-normalization/src/test/resources/test.conf
+++ b/data-pipeline-flink/de-normalization/src/test/resources/test.conf
@@ -3,8 +3,9 @@ include "base-test.conf"
 kafka {
   input.telemetry.topic = "flink.telemetry.unique"
   input.summary.topic = "flink.telemetry.derived"
-  output.success.topic = "flink.telemetry.denorm"
-  output.summary.topic = "flink.telemetry.derived.unique"
+  telemetry.denorm.output.topic = "flink.telemetry.denorm"
+  summary.denorm.output.topic = "flink.druid.events.summary"
+  summary.unique.events.topic = "flink.telemetry.derived.unique"
   output.failed.topic = "flink.telemetry.failed"
   output.duplicate.topic = "flink.telemetry.duplicate"
   groupId = "flink-telemetry-denorm-group"
@@ -12,12 +13,15 @@ kafka {
 
 telemetry.ignore.period.months = 4
 task {
-  denorm.parallelism = 1
-  denorm.sink.parallelism = 1
-  summary.sink.parallelism = 1
-  denorm.summary-dedup.parallelism = 1
+  window.count = 1
+  window.shards = 10
+  telemetry.downstream.operators.parallelism = 1
+  summary.downstream.operators.parallelism = 1
 }
 
+skip.events = ["INTERRUPT"]
+permit.eid=["AUDIT"]
+
 redis {
   host = 127.0.0.1
   port = 6341
@@ -32,8 +36,16 @@ redis-meta {
   port = 6341
   database {
     devicestore.id = 2
-    userstore.id = 4
+    userstore.id = 12
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  device.host = "localhost"
+  user.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6341
+  device.port = 6341
+  user.port = 6341
+  dialcode.port = 6341
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/test/resources/test2.conf b/data-pipeline-flink/de-normalization/src/test/resources/test2.conf
index b06010376f..0ee9d378e2 100644
--- a/data-pipeline-flink/de-normalization/src/test/resources/test2.conf
+++ b/data-pipeline-flink/de-normalization/src/test/resources/test2.conf
@@ -3,20 +3,24 @@ include "base-test.conf"
 kafka {
   input.telemetry.topic = "flink.telemetry.unique"
   input.summary.topic = "flink.telemetry.derived"
-  output.success.topic = "flink.telemetry.denorm"
-  output.summary.topic = "flink.telemetry.derived.unique"
+  telemetry.denorm.output.topic = "flink.telemetry.denorm"
+  summary.denorm.output.topic = "flink.druid.events.summary"
+  summary.unique.events.topic = "flink.telemetry.derived.unique"
   output.failed.topic = "flink.telemetry.failed"
   output.duplicate.topic = "flink.telemetry.duplicate"
   groupId = "flink-telemetry-denorm-group"
 }
 
 task {
-  denorm.parallelism = 1
-  denorm.sink.parallelism = 1
-  summary.sink.parallelism = 1
-  denorm.summary-dedup.parallelism = 1
+  window.count = 1
+  window.shards = 10
+  telemetry.downstream.operators.parallelism = 1
+  summary.downstream.operators.parallelism = 1
 }
 
+skip.events = ["INTERRUPT"]
+permit.eid=["AUDIT"]
+
 redis {
   host = 127.0.0.1
   port = 6340
@@ -31,10 +35,18 @@ redis-meta {
   port = 6340
   database {
     devicestore.id = 2
-    userstore.id = 4
+    userstore.id = 12
     contentstore.id = 5
     dialcodestore.id = 6
   }
+  content.host = "localhost"
+  device.host = "localhost"
+  user.host = "localhost"
+  dialcode.host = "localhost"
+  content.port = 6340
+  device.port = 6340
+  user.port = 6340
+  dialcode.port = 6340
 }
 
 telemetry.ignore.period.months = 6
diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index 72f0932fe7..56e2b35597 100644
--- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -1,20 +1,32 @@
 package org.sunbird.dp.fixture
 
+import java.util
+
+import com.google.gson.Gson
 import org.joda.time.DateTime
 
 object EventFixture {
-  
-  
+
+  val gson = new Gson()
   val deviceCacheData1 = """{"country":"India","state_custom":"Karnataka","devicespec":"{}","city":"Bengaluru","uaspec":"{}","district_custom":"BENGALURU URBAN SOUTH","country_code":"IN","firstaccess":"1571999041881","state_code_custom":"29"}"""
   val deviceCacheData2 = """{"user_declared_state":"Maharashtra","state_custom":"Maharashtra","devicespec":"{\"scrn\":\"5.46\",\"camera\":\"\",\"idisk\":\"25.44\",\"os\":\"Android 9\",\"id\":\"45f32f48592cb9bcf26bef9178b7bd20abe24932\",\"sims\":\"-1\",\"cpu\":\"abi: armeabi-v7a processor\t: 0 \",\"webview\":\"79.0.3945.116\",\"edisk\":\"25.42\",\"make\":\"Samsung SM-J400F\"}","uaspec":"{\"agent\":\"UNKNOWN\",\"ver\":\"UNKNOWN\",\"system\":\"Android\",\"raw\":\"Dalvik/2.1.0 (Linux U Android 9 SM-J400F Build/PPR1.180610.011)\"}","city":"Mumbai","country_code":"IN","firstaccess":"1578972432419","country":"India","country_name":"India","state":"Maharashtra","continent_name":"Asia","state_code":"MH","fcm_token":"d3ddT88xXLI:APA91bF9lJ4eH8tshAPgKiiZ3hL3sbib0pUN2I388T58oFDxUBQ2WKKuvtBga6iKiOPrgssNKLs4QBjZxE_BbtdGdO0gPdFPataEeXshgYxMKC0VT-oyjrNIZKdKkybQoyichBCiokTD","producer":"sunbirddev.diksha.app","district_custom":"Mumbai","user_declared_district":"Raigad","state_code_custom":"27"}"""
+  // device data without user declared location fields
+  val deviceCacheData3 = """{"state_custom":"Maharashtra","devicespec":"{\"scrn\":\"5.46\",\"camera\":\"\",\"idisk\":\"25.44\",\"os\":\"Android 9\",\"id\":\"45f32f48592cb9bcf26bef9178b7bd20abe24932\",\"sims\":\"-1\",\"cpu\":\"abi: armeabi-v7a processor\t: 0 \",\"webview\":\"79.0.3945.116\",\"edisk\":\"25.42\",\"make\":\"Samsung SM-J400F\"}","uaspec":"{\"agent\":\"UNKNOWN\",\"ver\":\"UNKNOWN\",\"system\":\"Android\",\"raw\":\"Dalvik/2.1.0 (Linux U Android 9 SM-J400F Build/PPR1.180610.011)\"}","city":"Mumbai","country_code":"IN","firstaccess":"1578972432419","country":"India","country_name":"India","state":"Maharashtra","continent_name":"Asia","state_code":"MH","fcm_token":"d3ddT88xXLI:APA91bF9lJ4eH8tshAPgKiiZ3hL3sbib0pUN2I388T58oFDxUBQ2WKKuvtBga6iKiOPrgssNKLs4QBjZxE_BbtdGdO0gPdFPataEeXshgYxMKC0VT-oyjrNIZKdKkybQoyichBCiokTD","producer":"sunbirddev.diksha.app","district_custom":"Mumbai","state_code_custom":"27"}"""
   
   val userCacheData1 = """{"usersignintype":"Anonymous","usertype":"TEACHER"}"""
   val userCacheData2 = """{"channel":"KV123","phoneverified":false,"createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":["English"],"email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":{},"userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"TEACHER","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":[],"grade":[],"roles":["BOOK_REVIEWER"],"status":1,"webpages":[],"createddate":"2019-04-11 08:58:16:512+0000","emailverified":true,"isdeleted":false,"locationids":[],"maskedemail":"a5**@yopmail.com","profilevisibility":{},"loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad"}"""
-  
+  val userCacheData3 = """{"channel":"KV123","phoneverified":"false","createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":"[\"English\"]","email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":"{}","userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"administrator", "usersubtype":"deo,hm","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":"","grade":"","roles":"[\"BOOK_REVIEWER\"]","status":"1","webpages":"[]","createddate":"2019-04-11 08:58:16:512+0000","emailverified":"true","isdeleted":"false","locationids":"[\"location-1\",\"location-2\",\"location-3\"]","maskedemail":"a5**@yopmail.com","profilevisibility":"{}","loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad","cluster":"Cluster001","block":"Sri Sai ACC Block","schoolname":"\\[RPMMAT M.S UDHADIH"}"""
+  val userCacheDataMap1: util.HashMap[String, String] = gson.fromJson(userCacheData1, new util.HashMap[String, String]().getClass)
+  val userCacheDataMap2: util.HashMap[String, String] = gson.fromJson(userCacheData3, new util.HashMap[String, String]().getClass)
+
   val contentCacheData1 = """{"code":"org.ekstep.literacy.story.21797","keywords":["Story"],"subject":["English"],"channel":"in.ekstep","description":"Write a short description of your lesson","language":["English"],"mimeType":"application/vnd.ekstep.ecml-archive","idealScreenSize":"normal","createdOn":"2018-04-26T09:59:27.336+0000","objectType":"Content","gradeLevel":["Class 9"],"appId":"ekstep_portal","contentDisposition":"inline","contentEncoding":"gzip","lastUpdatedOn":1571999041881,"lastSubmittedOn":"2018-06-15T13:06:56.090+0000","lastPublishedOn":1571999041881,"contentType":"Resource","lastUpdatedBy":"7224","identifier":"do_31249064359802470412856","audience":["Learner"],"creator":"Pravin Panpatil","os":["All"],"visibility":"Default","consumerId":"62e15662-bb09-439f-86e2-d65bd84f3c23","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","versionKey":"1524736767336","idealScreenDensity":"hdpi","framework":"NCF","createdBy":"7224","compatibilityLevel":1.0,"domain":["literacy"],"name":"Walk a little slower","board":"State (Maharashtra)","resourceType":"Read","status":"Draft","node_id":310297.0,"license":"CC BY 4.0","copyright":"Ekstep","author":"Ekstep","copyrightYear":2019.0}"""
   val contentCacheData2 = """{"identifier":"do_312526125187809280139353","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}"""
   val contentCacheData3 = """{"code":"do_312526125187809280139355","channel":"in.ekstep","downloadUrl":"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/14681653089315c3173a3e1.mp3","language":["English"],"mimeType":"application/octet-stream","idealScreenSize":"normal","createdOn":"2016-07-10T15:41:49.111+0000","objectType":"Content","gradeLevel":["Class 1"],"contentDisposition":"inline","contentEncoding":"identity","lastUpdatedOn":"2017-03-10T18:10:00.448+0000","contentType":"Asset","identifier":"do_312526125187809280139355","os":["All"],"visibility":"Default","mediaType":"audio","osId":"org.ekstep.launcher","ageGroup":["5-6"],"graph_id":"domain","nodeType":"DATA_NODE","pkgVersion":1.0,"versionKey":"1489169400448","license":"CC BY 4.0","idealScreenDensity":"hdpi","framework":"NCF","compatibilityLevel":1.0,"name":"do_312526125187809280139355","status":"Live","node_id":65545.0,"size":677510.0}"""
-  
+  val contentCacheData4 = """{"code":"org.sunbird.XiKRjo","channel":"0123221617357783046602","description":"गणित तीसरी कक्षा","language":["English"],"mimeType":"application/vnd.ekstep.content-collection","medium":["Hindi"],"idealScreenSize":"normal","createdOn":"2018-06-12T08:26:22.841+0000","objectType":"Content","gradeLevel":["Class 3"],"children":["do_312523863926120448117914","do_312523863926120448117918","do_312523863926120448117917","do_312523863926120448117916","do_312523863926120448117915","do_312523863926128640117922","do_312523863926128640117921","do_312523863926128640117920","do_312523863926128640117919","do_312523863926136832117926","do_312523863926136832117925","do_312523863926136832117924","do_312523863926128640117923","do_312523863926153216117930","do_312523863926145024117929","do_312523863926145024117928","do_312523863926145024117927","do_312523863926161408117934","do_312523863926161408117933","do_312523863926153216117932","do_312523863926153216117931","do_312523863926169600117938","do_312523863926161408117937","do_312523863926161408117936","do_312523863926161408117935","do_312523863926177792117942","do_312523863926169600117941","do_312523863926169600117940","do_312523863926169600117939","do_312523863926177792117946","do_312523863926177792117945","do_312523863926177792117944","do_312523863926177792117943","do_312523863926185984117950","do_312523863926185984117949","do_312523863926185984117948","do_312523863926185984117947","do_312523863926194176117954","do_312523863926194176117953","do_312523863926194176117952","do_312523863926185984117951","do_312523863926202368117958","do_312523863926202368117957","do_312523863926202368117956","do_312523863926194176117955","do_312523863926210560117962","do_312523863926210560117961","do_312523863926210560117960","do_312523863926202368117959","do_312523863926218752117966","do_312523863926218752117965","do_312523863926218752117964","do_312523863926210560117963","do_312523863926226944117970","do_312523863926226944117969","do_312523863926218752117968","do_312523863926218752117967","do_312523863926235136117974","do_312523863926226944117973","do_312523863926226944117972","do_312523863926226944117971"],"appId":"prod.diksha.portal","contentDisposition":"inline","contentEncoding":"gzip","lastUpdatedOn":"2018-06-12T08:31:38.614+0000","contentType":"TextBook","identifier":"do_312523863923441664117896","createdFor":["0123221617357783046602"],"audience":["Learner"],"os":["All"],"visibility":"Default","consumerId":"0aa13c48-dda0-4259-9007-c795dacd7b9c","mediaType":"content","osId":"org.ekstep.quiz.app","graph_id":"domain","nodeType":"DATA_NODE","versionKey":"1528792298614","idealScreenDensity":"hdpi","framework":"mh_k-12_1","createdBy":"80ae760d-f9dd-43a9-91a7-09dfabba2293","compatibilityLevel":1.0,"name":"test","board":"State (Maharashtra)","resourceType":"Book","status":"Retired","node_id":398184.0,"license":"CC BY 4.0","copyright":"MITRA","author":"MITRA","copyrightYear":2019.0}"""
+  val contentCacheData5 = """{"identifier":"7426472e-8b1a-4387-8b7a-962cb6cda006","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}"""
+
+  val collectionCache1 = """{"identifier":"do_31331086175718604812701","code":"c361b157-408c-4347-9be3-38d9d4ea2b90","visibility":"Parent","description":"Anction Words","mimeType":"application/vnd.ekstep.content-collection","graph_id":"domain","nodeType":"DATA_NODE","createdOn":"2018-06-15T13:06:56.090+0000","versionKey":"1529068016090","objectType":"Content","dialcodes":["TNPF7T"],"collections":["do_312526125186424832139255"],"name":"Anction Words","lastUpdatedOn":"2018-06-15T13:06:56.090+0000","lastsubmittedon":1571999041881,"lastPublishedOn":"2018-06-15T13:06:56.090+0000","contentType":"TextBookUnit","status":"Draft","node_id":438622.0,"license":"CC BY 4.0"}"""
+
   val dialcodeCacheData1 = """{"identifier":"GWNI38","batchcode":"jkpublisher.20180801T134105","channel":"01254592085869363222","dialcode_index":2328993,"generated_on":"2018-08-01T13:41:53.695","published_on":1571999041881,"publisher":"jkpublisher","status":"Draft"}"""
   val dialcodeCacheData2 = """{"identifier":"PCZKA3","batchcode":"jkpublisher.20180801T122031","channel":"01254592085869363222","dialcode_index":1623464,"generated_on":1571999041881,"published_on":"2018-08-01T13:41:53.695Z","publisher":"jkpublisher","status":"Draft"}"""
   
@@ -22,6 +34,15 @@ object EventFixture {
   val currentDate: Long = DateTime.now().getMillis
   val olderDate: Long = DateTime.now().minusMonths(5).getMillis
   val futureDate: Long = DateTime.now().plusMonths(1).getMillis
+
+  val telemetryEvent: String = s"""{"actor":{"type":"User","id":"b7470841-7451-43db-b5c7-2dcf4f8d3b23"},"eid":"INTERACT",
+                                  |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}},
+                                  |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate,
+                                  |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d",
+                                  |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272",
+                                  |"did":"45f32f48592cb9bcf26bef9178b7bd20abe24932"},"flags":{"dd_processed":true},
+                                  |"mid":"mid1","type":"events","object":{"id":"","type":"",
+                                  |"version":"","rollup":{}}}""".stripMargin
   
   val telemetrEvents: List[String] = List(
       
@@ -96,7 +117,7 @@ object EventFixture {
       |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272",
       |"did":""},"flags":{"dd_processed":true},
       |"mid":"mid8","type":"events","object":{"id":"do_312526125187809280139352","type":"Content",
-      |"version":"","rollup":{"l1":"do_312526125187809280139353"}}}""".stripMargin,
+      |"version":"","rollup":{"l1":"do_312526125187809280139353","l2":"do_312523863923441664117896"}}}""".stripMargin,
       
       // Future date
       s"""{"actor":{"type":"System","id":"anonymous"},"eid":"INTERACT",
@@ -126,8 +147,68 @@ object EventFixture {
       |"cdata":[],"rollup":{"l1":"505c7c48ac6dc1edc9b08f21db5a571d"}},"object":{"id":"do_31249064359802470412856","type":"content","ver":"","rollup":{}},
       |"tags":["505c7c48ac6dc1edc9b08f21db5a571d"],"edata":{"state":"COMPLETED","prevstate":"INPROGRESS",
       |"props":["stats.downloadedSize","status","updatedOn"],"duration":0.8},"syncts":1.58156611767E12,"@timestamp":"2020-02-13T03:55:17.670Z",
-      |"contentdata":{"objectType":"Content"},"userdata":{"firstname":"A512"},"type":"events"}""".stripMargin
-            
+      |"contentdata":{"objectType":"Content"},"userdata":{"firstname":"A512"},"type":"events"}""".stripMargin,
+
+      s"""{"actor":{"type":"User","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc82"},"eid":"INTERRUPT",
+       |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}},
+       |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate,
+       |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d",
+       |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272",
+       |"did":"45f32f48592cb9bcf26bef9178b7bd20abe24932"},"flags":{"dd_processed":true},
+       |"mid":"mid6","type":"events","object":{"id":"PCZKA4","type":"qr",
+       |"version":"","rollup":{}}}""".stripMargin,
+
+      // event without did - INTERRUPT skipped
+      s"""{"actor":{"type":"User","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc82"},"eid":"INTERRUPT",
+       |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}},
+       |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate,
+       |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d",
+       |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272"},
+       |"flags":{"dd_processed":true},
+       |"mid":"mid11","type":"events","object":{"id":"PCZKA4","type":"qr",
+       |"version":"","rollup":{}}}""".stripMargin,
+
+      // event without did
+      s"""{"actor":{"type":"User","id":"b7470841-7451-43db-b5c7-2dcf4f8d3b23"},"eid":"INTERACT",
+       |"edata":{"type":"OTHER","subtype":"sheen-animation-ended","id":"library","pageid":"library","extra":{"pos":[]}},
+       |"ver":"3.0","syncts":1.579564974098E12,"@timestamp":"2020-01-21T00:02:54.098Z","ets":$currentDate,
+       |"context":{"cdata":[],"env":"home","channel":"505c7c48ac6dc1edc9b08f21db5a571d",
+       |"pdata":{"id":"sunbird.app","pid":"sunbird.app","ver":"2.3.144"},"sid":"df936f82-e982-41ec-8412-70d414458272"},
+       |"flags":{"dd_processed":true},
+       |"mid":"mid12","type":"events","object":{"id":"","type":"",
+       |"version":"","rollup":{}}}""".stripMargin,
+
+      // SUMMARY event - user_denorm = true, device_denorm=true, content_denorm=true
+      s"""{"eid":"SUMMARY","ets":$currentDate,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a",
+         |"actor":{"id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","type":"User"},
+         |"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0",
+         |"pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da",
+         |"did":"264d679186d4b0734d858d4e18d4d31e","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},
+         |{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],
+         |"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_31249064359802470412856","ver":"1","type":"Content","rollup":{}},
+         |"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,
+         |"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"},
+         |{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"},
+         |{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}}""".stripMargin,
+
+      // ME_DEVICE_SUMMARY event - should skip
+      s"""
+         |{"eid":"ME_DEVICE_SUMMARY","ets":$currentDate,"syncts":$currentDate,"ver":"1.0","mid":
+         |"CFBA22543AA3EAD4C2737931D34F2E8D","context":{"pdata":{"id":"AnalyticsDataPipeline","ver":"1.0",
+         |"model":"DeviceSummary"},"granularity":"DAY","date_range":{"from":1572786370125,"to":1572786403121}},
+         |"dimensions":{"did":"3eb8d5dc49b063650ca18920956ea04e","channel":"ROOT_ORG"},"edata":{"eks":{"firstAccess":
+         |1572786370121,"dial_stats":{"total_count":3,"success_count":3,"failure_count":0},"content_downloads":0,
+         |"contents_played":0,"total_ts":0.0,"total_launches":0,"unique_contents_played":0}}}
+         """.stripMargin,
+      //SB-25755: if enrol-complete , denorm the event
+      s"""
+          |{"actor":{"id":"7426472e-8b1a-4387-8b7a-962cb6cda006","type":"User"},"eid":"AUDIT","edata":{"props":["status","completedon"],
+          |"type":"enrol-complete"},"ver":"3.0","syncts":1626346241621,"ets":$currentDate,"context":{"channel":"in.sunbird",
+          |"env":"Course","sid":"2ba8a10a-9722-42c6-a27f-f9fd86ff6bb5","did":"0a65cbaf-2d0e-4cc8-bc70-6ee1e71fb605","pdata":
+          |{"ver":"3.0","id":"org.sunbird.learning.platform","pid":"course-progress-updater"},"cdata":[{"type":"CourseBatch","id":"01331092647782809655"},
+          |{"type":"Course","id":"do_31331086175718604812701"}]},"mid":"LP.AUDIT.d14d8be6-da4e-4ee9-b833-fd86d57b8808",
+          |"object":{"id":"7426472e-8b1a-4387-8b7a-962cb6cda006","type":"User","rollup":{"l1":"do_31331086175718604812701"}},"tags":[]}
+        """.stripMargin
   )
 
   val summaryEvents: List[String] = List(
diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala
new file mode 100644
index 0000000000..45cbab0c47
--- /dev/null
+++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/CheckDerivedLocationStreamTaskTestSpec.scala
@@ -0,0 +1,118 @@
+package org.sunbird.dp.spec
+
+import com.google.gson.Gson
+import com.typesafe.config.{Config, ConfigFactory}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
+import org.apache.flink.streaming.api.functions.sink.SinkFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+import org.apache.flink.test.util.MiniClusterWithClientResource
+import org.mockito.Mockito
+import org.mockito.Mockito.when
+import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.denorm.domain.Event
+import org.sunbird.dp.denorm.task.{DenormalizationConfig, DenormalizationStreamTask}
+import org.sunbird.dp.fixture.EventFixture
+import redis.embedded.RedisServer
+
+import java.util
+
+class CheckDerivedLocationStreamTaskTestSpec extends BaseTestSpec {
+  implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
+
+  val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
+    .setConfiguration(testConfiguration())
+    .setNumberSlotsPerTaskManager(1)
+    .setNumberTaskManagers(1)
+    .build)
+
+  var redisServer: RedisServer = _
+  val config: Config = ConfigFactory.load("test.conf")
+  val denormConfig: DenormalizationConfig = new DenormalizationConfig(config, "DenormTest")
+  val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
+  val gson = new Gson()
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    redisServer = new RedisServer(6341)
+    redisServer.start()
+
+    BaseMetricsReporter.gaugeMetrics.clear()
+
+    setupRedisTestData()
+    flinkCluster.before()
+  }
+
+  override protected def afterAll(): Unit = {
+    super.afterAll()
+    redisServer.stop()
+    flinkCluster.after()
+  }
+
+  def setupRedisTestData() {
+
+    val redisConnect = new RedisConnect(denormConfig.metaRedisHost, denormConfig.metaRedisPort, denormConfig)
+
+    // Insert device test data
+    var jedis = redisConnect.getConnection(denormConfig.deviceStore)
+    jedis.hmset("45f32f48592cb9bcf26bef9178b7bd20abe24932", gson.fromJson(EventFixture.deviceCacheData3, new util.HashMap[String, String]().getClass))
+    jedis.close()
+
+  }
+
+  "De-normalization pipeline" should "denormalize location metadata properly for blank user declared scenario" in {
+
+    when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.telemetryInputTopic)).thenReturn(new InputSource1)
+    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.telemetryDenormOutputTopic)).thenReturn(new DenormEventsSink1)
+
+    val task = new DenormalizationStreamTask(denormConfig, mockKafkaUtil)
+    task.process()
+
+    val event = DenormEventsSink1.values("mid1")
+    event.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true)
+    // derived location should be from ip-resolved
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("district") should be("Mumbai")
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("state") should be("Maharashtra")
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("from") should be("ip-resolved")
+
+    val device3Data = event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]]
+    device3Data.get("firstaccess") should be (1578972432419L)
+    val deviceSpecData = device3Data.get("devicespec").asInstanceOf[util.Map[String, Any]]
+    deviceSpecData.get("edisk") should be ("25.42")
+    deviceSpecData.get("make") should be ("Samsung SM-J400F")
+    // user declared is empty
+    val userDeclaredLocationData = device3Data.get("userdeclared").asInstanceOf[util.Map[String, Any]]
+    userDeclaredLocationData.get("district") should be ("")
+    userDeclaredLocationData.get("state") should be ("")
+
+  }
+
+}
+
+class InputSource1 extends SourceFunction[Event] {
+
+  override def run(ctx: SourceContext[Event]) {
+    val gson = new Gson()
+    val eventMap = gson.fromJson(EventFixture.telemetryEvent, new util.HashMap[String, Any]().getClass)
+    ctx.collect(new Event(eventMap))
+  }
+
+  override def cancel() = {}
+}
+
+class DenormEventsSink1 extends SinkFunction[Event] {
+
+  override def invoke(event: Event): Unit = {
+    synchronized {
+      DenormEventsSink1.values.put(event.mid(), event)
+    }
+  }
+}
+
+object DenormEventsSink1 {
+  val values: scala.collection.mutable.Map[String, Event] = scala.collection.mutable.Map[String, Event]()
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala
index 3bf3477734..7fd2533d0f 100644
--- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala
+++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/DenormalizationStreamTaskTestSpec.scala
@@ -21,6 +21,8 @@ import org.sunbird.dp.fixture.EventFixture
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
 import redis.embedded.RedisServer
 
+import scala.collection.JavaConverters._
+
 class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
 
   implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
@@ -66,8 +68,8 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
 
     // Insert user test data
     jedis = redisConnect.getConnection(denormConfig.userStore)
-    jedis.set("b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheData1)
-    jedis.set("610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheData2)
+    jedis.hmset(denormConfig.userStoreKeyPrefix + "b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheDataMap1)
+    jedis.hmset(denormConfig.userStoreKeyPrefix + "610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheDataMap2)
     jedis.close()
 
     // Insert dialcode test data
@@ -81,6 +83,9 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
     jedis.set("do_31249064359802470412856", EventFixture.contentCacheData1)
     jedis.set("do_312526125187809280139353", EventFixture.contentCacheData2)
     jedis.set("do_312526125187809280139355", EventFixture.contentCacheData3)
+    jedis.set("do_312523863923441664117896", EventFixture.contentCacheData4)
+    jedis.set("7426472e-8b1a-4387-8b7a-962cb6cda006", EventFixture.contentCacheData5)
+    jedis.set("do_31331086175718604812701", EventFixture.collectionCache1)
     jedis.close()
 
   }
@@ -88,21 +93,26 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
   "De-normalization pipeline" should "denormalize content, user, device and location metadata" in {
 
     when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.telemetryInputTopic)).thenReturn(new InputSource)
-    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.denormSuccessTopic)).thenReturn(new DenormEventsSink)
+    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.telemetryDenormOutputTopic)).thenReturn(new DenormEventsSink)
 
     val task = new DenormalizationStreamTask(denormConfig, mockKafkaUtil)
     task.process()
-    DenormEventsSink.values.size should be (10)
+
+    DenormEventsSink.values.size should be (13)
     DenormEventsSink.values.get("mid10") should be (None)
 
     var event = DenormEventsSink.values("mid1")
     event.kafkaKey() should be ("758e054a400f20f7677f2def76427dc13ad1f837")
-
     event.flags().get("device_denorm").asInstanceOf[Boolean] should be (false)
     event.flags().get("user_denorm").asInstanceOf[Boolean] should be (true)
     Option(event.flags().get("dialcode_denorm")) should be (None)
     Option(event.flags().get("content_denorm")) should be (None)
     Option(event.flags().get("location_denorm")) should be (None)
+
+    val user1Data = event.getMap().get("userdata").asInstanceOf[util.Map[String, Any]]
+    user1Data.get("usersignintype") should be("Anonymous")
+    user1Data.get("usertype") should be("TEACHER")
+    user1Data.get("userlogintype") should be("NA")
     
     event = DenormEventsSink.values("mid2")
     event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true)
@@ -113,14 +123,54 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
     Option(event.flags().get("coll_denorm")) should be (None)
 
     event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("lastsubmittedon") should be(1529068016090L)
-    
+    event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("channel") should be("in.ekstep")
+    event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("lastpublishedon") should be(1.571999041881E12)
+    event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Resource")
+    event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]].get("keywords").asInstanceOf[util.ArrayList[String]].get(0) should be ("Story")
+
+    event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("statecustomcode") should be("29")
+    event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("countrycode") should be("IN")
+    event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("firstaccess") should be(1571999041881L)
+    event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]].get("districtcustom") should be("BENGALURU URBAN SOUTH")
+
+    val user2Data = event.getMap().get("userdata").asInstanceOf[util.Map[String, Any]]
+    user2Data.get("userlogintype") should be("Student")
+    user2Data.get("usersignintype") should be("Self-Signed-In")
+    user2Data.get("usertype") should be("administrator")
+    user2Data.get("usersubtype") should be("deo,hm")
+    user2Data.get("subject").asInstanceOf[util.List[String]].asScala should be(List("English"))
+    user2Data.get("state") should be("Telangana")
+    user2Data.get("cluster") should be("Cluster001")
+    user2Data.get("schoolname") should be("[RPMMAT M.S UDHADIH")
+    user2Data.get("block") should be ("Sri Sai ACC Block")
+
     event = DenormEventsSink.values("mid3")
     event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true)
     event.flags().get("user_denorm").asInstanceOf[Boolean] should be (false)
     event.flags().get("content_denorm").asInstanceOf[Boolean] should be (true)
     event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true)
     event.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true)
-    
+
+    event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Asset")
+    Option(event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contentType")) should be (None)
+    event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("contenttype") should be("Asset")
+    event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("framework") should be("NCF")
+    event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("name") should be("do_312526125187809280139355")
+    event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]].get("lastupdatedon") should be(1489169400448L)
+
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("district") should be("Raigad")
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("state") should be("Maharashtra")
+    event.getMap().get("derivedlocationdata").asInstanceOf[util.Map[String, Any]].get("from") should be("user-declared")
+
+    val device3Data = event.getMap().get("devicedata").asInstanceOf[util.Map[String, Any]]
+    device3Data.get("firstaccess") should be (1578972432419L)
+    val deviceSpecData = device3Data.get("devicespec").asInstanceOf[util.Map[String, Any]]
+    deviceSpecData.get("edisk") should be ("25.42")
+    deviceSpecData.get("make") should be ("Samsung SM-J400F")
+    val userDeclaredLocationData = device3Data.get("userdeclared").asInstanceOf[util.Map[String, Any]]
+    userDeclaredLocationData.get("district") should be ("Raigad")
+    userDeclaredLocationData.get("state") should be ("Maharashtra")
+
     event = DenormEventsSink.values("mid4")
     event.flags().get("device_denorm").asInstanceOf[Boolean] should be (true)
     event.flags().get("user_denorm").asInstanceOf[Boolean] should be (true)
@@ -137,26 +187,63 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
     Option(event.flags().get("content_denorm")) should be (None)
     Option(event.flags().get("location_denorm")) should be (None)
 
+    event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("batchcode") should be("jkpublisher.20180801T122031")
+    event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("channel") should be("01254592085869363222")
+    event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("generatedon") should be(1.571999041881E12)
+    event.getMap().get("dialcodedata").asInstanceOf[util.Map[String, Any]].get("publishedon") should be(1533130913695L)
+
     // TODO: Complete the assertions
     event = DenormEventsSink.values("mid6")
     event = DenormEventsSink.values("mid7")
     event = DenormEventsSink.values("mid8")
+    event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true)
+    event.flags().get("l2_denorm").asInstanceOf[Boolean] should be (true)
+    val l2Data = event.getMap().get("l2data").asInstanceOf[util.Map[String, Any]]
+    l2Data should not be null
+
+    l2Data.get("contenttype") should be("TextBook")
+    l2Data.get("mimetype") should be("application/vnd.ekstep.content-collection")
+    l2Data.get("contenttype") should be("TextBook")
+    l2Data.get("channel") should be("0123221617357783046602")
+    l2Data.get("board") should be("State (Maharashtra)")
+    l2Data.get("name") should be("test")
+    l2Data.get("framework") should be("mh_k-12_1")
     event = DenormEventsSink.values("mid9")
 
+    event = DenormEventsSink.values("LP.AUDIT.d14d8be6-da4e-4ee9-b833-fd86d57b8808")
+    event.flags().get("device_denorm").asInstanceOf[Boolean] should be (false)
+    event.flags().get("user_denorm").asInstanceOf[Boolean] should be (false)
+    event.flags().get("coll_denorm").asInstanceOf[Boolean] should be (true)
+    event.flags().get("content_denorm").asInstanceOf[Boolean] should be (true)
+
+    val contentData = event.getMap().get("contentdata").asInstanceOf[util.Map[String, Any]]
+    contentData.get("name") should be ("Anction Words")
+    contentData.get("mimetype") should be ("application/vnd.ekstep.content-collection")
+    contentData.get("objecttype") should be ("Content")
+    contentData.get("contenttype") should be ("TextBookUnit")
+    contentData.get("status") should be ("Draft")
+
+    val collData = event.getMap().get("collectiondata").asInstanceOf[util.Map[String, Any]]
+    collData.get("name") should be ("Anction Words")
+    collData.get("mimetype") should be ("application/vnd.ekstep.content-collection")
+    collData.get("objecttype") should be ("Content")
+    collData.get("contenttype") should be ("TextBookUnit")
+    collData.get("status") should be ("Draft")
+
     // Location Denorm Metrics Assertion
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (7)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (3)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locTotal}").getValue() should be (10)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (8)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (5)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locTotal}").getValue() should be (13)
 
     // Content Denorm Metrics Assertion
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheHit}").getValue() should be (4)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheMiss}").getValue() should be (3)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (7)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheHit}").getValue() should be (6)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentCacheMiss}").getValue() should be (2)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (8)
 
     // User Denorm Metrics Assertion
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (3)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (4)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (7)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (5)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (5)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (10)
 
     // Dialcode Denorm Metrics Assertion
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeCacheHit}").getValue() should be (2)
@@ -164,11 +251,12 @@ class DenormalizationStreamTaskTestSpec extends BaseTestSpec {
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeTotal}").getValue() should be (3)
 
     // Device Denorm Metrics Assertion
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheHit}").getValue() should be (7)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheMiss}").getValue() should be (2)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceTotal}").getValue() should be (9)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheHit}").getValue() should be (8)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceCacheMiss}").getValue() should be (3)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.deviceTotal}").getValue() should be (11)
 
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.eventsExpired}").getValue() should be (1)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.eventsSkipped}").getValue() should be (3) // Skipped INTERRUPT event
 
   }
   
diff --git a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala
index 99db935d88..8f55181288 100644
--- a/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala
+++ b/data-pipeline-flink/de-normalization/src/test/scala/org/sunbird/dp/spec/SummaryDenormalizationStreamTaskTestSpec.scala
@@ -21,6 +21,8 @@ import org.sunbird.dp.fixture.EventFixture
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
 import redis.embedded.RedisServer
 
+import scala.collection.JavaConverters._
+
 class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec {
 
   implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
@@ -66,8 +68,8 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec {
 
     // Insert user test data
     jedis = redisConnect.getConnection(denormConfig.userStore)
-    jedis.set("b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheData1)
-    jedis.set("610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheData2)
+    jedis.hmset(denormConfig.userStoreKeyPrefix + "b7470841-7451-43db-b5c7-2dcf4f8d3b23", EventFixture.userCacheDataMap1)
+    jedis.hmset(denormConfig.userStoreKeyPrefix + "610bab7d-1450-4e54-bf78-c7c9b14dbc81", EventFixture.userCacheDataMap2)
     jedis.close()
 
     // Insert dialcode test data
@@ -88,8 +90,8 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec {
   "Summary Denormalization pipeline" should "denormalize content, user, device and location metadata for summary events" in {
 
     when(mockKafkaUtil.kafkaEventSource[Event](denormConfig.summaryInputTopic)).thenReturn(new SummaryInputSource)
-    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.denormSuccessTopic)).thenReturn(new SummaryDenormEventsSink)
-    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryOutputEventsTopic)).thenReturn(new SummaryEventsSink)
+    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryDenormOutputTopic)).thenReturn(new SummaryDenormEventsSink)
+    when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.summaryUniqueEventsTopic)).thenReturn(new SummaryEventsSink)
     when(mockKafkaUtil.kafkaEventSink[Event](denormConfig.duplicateTopic)).thenReturn(new DuplicateEventsSink)
 
     val task = new SummaryDenormalizationStreamTask(denormConfig, mockKafkaUtil)
@@ -102,19 +104,35 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec {
     event1.kafkaKey() should be ("45f32f48592cb9bcf26bef9178b7bd20abe24932")
 
     event1.flags().get("device_denorm").asInstanceOf[Boolean] should be (true)
-    event1.flags().get("user_denorm").asInstanceOf[Boolean] should be (false)
+    event1.flags().get("user_denorm").asInstanceOf[Boolean] should be (true)
     Option(event1.flags().get("dialcode_denorm")) should be (None)
     Option(event1.flags().get("content_denorm")) should be (None)
     Option(event1.flags().get("location_denorm")) should be (None)
+    val event1UserData = event1.getTelemetry.read[util.HashMap[String, AnyRef]]("userdata").getOrElse(new util.HashMap()).asScala
+    event1UserData("usersignintype").asInstanceOf[String] should be ("Anonymous")
+    event1UserData("userlogintype").asInstanceOf[String] should be ("NA")
+    event1UserData("usertype").asInstanceOf[String] should be ("TEACHER")
 
     val event2 = SummaryDenormEventsSink.values("mid2")
     event2.flags().get("device_denorm").asInstanceOf[Boolean] should be (true)
-    event2.flags().get("user_denorm").asInstanceOf[Boolean] should be (false)
+    event2.flags().get("user_denorm").asInstanceOf[Boolean] should be (true)
     Option(event2.flags().get("dialcode_denorm")) should be (None)
     event2.flags().get("content_denorm").asInstanceOf[Boolean] should be (true)
     event2.flags().get("loc_denorm").asInstanceOf[Boolean] should be (true)
     Option(event2.flags().get("coll_denorm")) should be (Some(true))
 
+    val event2UserData = event2.getTelemetry.read[util.HashMap[String, AnyRef]]("userdata").getOrElse(new util.HashMap()).asScala
+    event2UserData("usersignintype") should be ("Self-Signed-In")
+    event2UserData("userlogintype") should be ("Student")
+    event2UserData("usertype") should be ("administrator")
+    event2UserData("usersubtype") should be ("deo,hm")
+    event2UserData("subject").asInstanceOf[util.List[String]].asScala should be(List("English"))
+    event2UserData("cluster") should be ("Cluster001")
+    event2UserData("schoolname") should be ("[RPMMAT M.S UDHADIH")
+    event2UserData("block") should be ("Sri Sai ACC Block")
+
+    val event2ContentData = event2.getTelemetry.read[util.HashMap[String, AnyRef]]("contentdata").getOrElse(new util.HashMap()).asScala
+    event2ContentData.get("keywords").get.asInstanceOf[util.ArrayList[String]].get(0) should be ("Story")
 
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheHit}").getValue() should be (2)
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.locCacheMiss}").getValue() should be (0)
@@ -126,9 +144,9 @@ class SummaryDenormalizationStreamTaskTestSpec extends BaseTestSpec {
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.contentTotal}").getValue() should be (1)
 
     // User Denorm Metrics Assertion
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (0)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheHit}").getValue() should be (2)
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userCacheMiss}").getValue() should be (0)
-    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (0)
+    BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.userTotal}").getValue() should be (2)
 
     // Dialcode Denorm Metrics Assertion
     BaseMetricsReporter.gaugeMetrics(s"${denormConfig.jobName}.${denormConfig.dialcodeCacheHit}").getValue() should be (0)
diff --git a/data-pipeline-flink/device-profile-updater/pom.xml b/data-pipeline-flink/device-profile-updater/pom.xml
index d047f265cf..a10f9b54c0 100644
--- a/data-pipeline-flink/device-profile-updater/pom.xml
+++ b/data-pipeline-flink/device-profile-updater/pom.xml
@@ -30,7 +30,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -54,13 +54,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -73,7 +73,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -107,6 +107,14 @@
     		<sourceDirectory>src/main/scala</sourceDirectory>
 		    <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -157,10 +165,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties b/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties
deleted file mode 100644
index dda4b21056..0000000000
--- a/data-pipeline-flink/device-profile-updater/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=telemetry-extractor.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/dp-core/pom.xml b/data-pipeline-flink/dp-core/pom.xml
index b1eb6d8442..cfe8164c5f 100644
--- a/data-pipeline-flink/dp-core/pom.xml
+++ b/data-pipeline-flink/dp-core/pom.xml
@@ -20,13 +20,13 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-connector-kafka_${scala.version}</artifactId>
+            <artifactId>flink-connector-kafka_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
         </dependency>
         <dependency>
@@ -49,6 +49,11 @@
             <artifactId>gson</artifactId>
             <version>2.4</version>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <version>2.10.0</version>
+        </dependency>
         <dependency>
             <groupId>redis.clients</groupId>
             <artifactId>jedis</artifactId>
@@ -78,20 +83,20 @@
         </dependency>
         <dependency>
             <groupId>net.manub</groupId>
-            <artifactId>scalatest-embedded-kafka_${scala.version}</artifactId>
+            <artifactId>scalatest-embedded-kafka_${scala.maj.version}</artifactId>
             <version>2.0.0</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_${scala.version}</artifactId>
+            <artifactId>kafka_${scala.maj.version}</artifactId>
             <version>2.4.1</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
@@ -122,7 +127,7 @@
         <dependency>
             <groupId>org.cassandraunit</groupId>
             <artifactId>cassandra-unit</artifactId>
-            <version>3.1.1.0</version>
+            <version>3.11.2.0</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
@@ -132,6 +137,14 @@
         <testSourceDirectory>src/test/scala</testSourceDirectory>
 
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.20</version>
@@ -186,10 +199,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/dp-core/src/main/resources/base-config.conf b/data-pipeline-flink/dp-core/src/main/resources/base-config.conf
index eed90dcebf..a7c74e02b3 100644
--- a/data-pipeline-flink/dp-core/src/main/resources/base-config.conf
+++ b/data-pipeline-flink/dp-core/src/main/resources/base-config.conf
@@ -1,8 +1,11 @@
 kafka {
-  broker-servers = "localhost:9092"
-  zookeeper = "localhost:2181"
+  consumer.broker-servers = "localhost:9092"
   producer {
+    broker-servers = "localhost:9092"
     max-request-size = 1572864
+    batch.size = 98304
+    linger.ms = 10
+    compression = "snappy"
   }
 }
 
@@ -22,11 +25,13 @@ job {
 }
 
 task {
-  parallelism = 1
-  consumer.parallelism = 1
+  checkpointing.compressed = true
   checkpointing.interval = 60000
+  checkpointing.pause.between.seconds = 30000
   restart-strategy.attempts = 3
   restart-strategy.delay = 30000 # in milli-seconds
+  parallelism = 1
+  consumer.parallelism = 1
 }
 
 redisdb.connection.timeout = 30000
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala
index f2e61fa512..89d380385d 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/cache/DataCache.scala
@@ -1,7 +1,6 @@
 package org.sunbird.dp.core.cache
 
 import java.util
-
 import com.google.gson.Gson
 import org.slf4j.LoggerFactory
 import org.sunbird.dp.core.job.BaseJobConfig
@@ -9,7 +8,9 @@ import redis.clients.jedis.Jedis
 import redis.clients.jedis.exceptions.{JedisConnectionException, JedisException}
 
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 import scala.collection.mutable.Map
+import scala.collection.immutable
 
 class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val dbIndex: Int, val fields: List[String]) {
 
@@ -25,31 +26,56 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d
     this.redisConnection.close()
   }
 
-  def hgetAllWithRetry(key: String): Map[String, String] = {
+  def hgetAllWithRetry(key: String): mutable.Map[String, AnyRef] = {
     try {
-      hgetAll(key)
+      convertToComplexDataTypes(hgetAll(key))
     } catch {
       case ex: JedisException =>
         logger.error("Exception when retrieving data from redis cache", ex)
         this.redisConnection.close()
         this.redisConnection = redisConnect.getConnection(dbIndex)
-        hgetAll(key)
+        convertToComplexDataTypes(hgetAll(key))
     }
+  }
 
+  def isArray(value: String): Boolean = {
+    val redisValue = value.trim
+    redisValue.length > 0 && redisValue.startsWith("[")
+  }
+
+   def isObject(value: String) = {
+     val redisValue = value.trim
+     redisValue.length > 0 && redisValue.startsWith("{")
+   }
+
+  def convertToComplexDataTypes(data: mutable.Map[String, String]): mutable.Map[String, AnyRef] = {
+    val result = mutable.Map[String, AnyRef]()
+    data.keys.map {
+      redisKey =>
+        val redisValue = data(redisKey)
+        if(isArray(redisValue)) {
+          result += redisKey -> gson.fromJson(redisValue, new util.ArrayList[AnyRef]().getClass)
+        } else if (isObject(redisValue)) {
+          result += redisKey -> gson.fromJson(redisValue, new util.HashMap[String, AnyRef]().getClass)
+        } else {
+          result += redisKey -> redisValue
+        }
+    }
+    result
   }
 
-  private def hgetAll(key: String): Map[String, String] = {
+  private def hgetAll(key: String): mutable.Map[String, String] = {
     val dataMap = redisConnection.hgetAll(key)
     if (dataMap.size() > 0) {
       if(fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
       dataMap.values().removeAll(util.Collections.singleton(""))
       dataMap.asScala
     } else {
-      Map[String, String]()
+      mutable.Map[String, String]()
     }
   }
 
-  def getWithRetry(key: String): Map[String, AnyRef] = {
+  def getWithRetry(key: String): mutable.Map[String, AnyRef] = {
     try {
       get(key)
     } catch {
@@ -62,15 +88,15 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d
 
   }
 
-  private def get(key: String): Map[String, AnyRef] = {
+  private def get(key: String): mutable.Map[String, AnyRef] = {
     val data = redisConnection.get(key)
-    if (data != null && !data.isEmpty()) {
+    if (data != null && !data.isEmpty) {
       val dataMap = gson.fromJson(data, new util.HashMap[String, AnyRef]().getClass)
       if(fields.nonEmpty) dataMap.keySet().retainAll(fields.asJava)
       dataMap.values().removeAll(util.Collections.singleton(""))
       dataMap.asScala
     } else {
-      Map[String, AnyRef]()
+      mutable.Map[String, AnyRef]()
     }
   }
 
@@ -116,6 +142,54 @@ class DataCache(val config: BaseJobConfig, val redisConnect: RedisConnect, val d
     redisConnection.set(key, value)
   }
 
+  def sMembers(key: String): util.Set[String] = {
+    redisConnection.smembers(key)
+  }
+  def getKeyMembers(key: String): util.Set[String] = {
+    try {
+      sMembers(key)
+    } catch {
+      case ex: JedisException =>
+        logger.error("Exception when retrieving data from redis cache", ex)
+        this.redisConnection.close()
+        this.redisConnection = redisConnect.getConnection(dbIndex)
+        sMembers(key)
+    }
+  }
+
+  def hdel(key: String, fieldSeq: Seq[String]): Unit = {
+    this.redisConnection.hdel(key, fieldSeq: _*)
+  }
+
+  def hdelWithRetry(key: String, fieldSeq: Seq[String]): Unit = {
+    try {
+      hdel(key, fieldSeq)
+    } catch {
+      case ex: JedisException =>
+        logger.error("Exception when deleting fields in hash", ex)
+        this.redisConnection.close()
+        this.redisConnection = redisConnect.getConnection(dbIndex)
+        hdel(key, fieldSeq)
+    }
+  }
+
+  def hIncBy(key: String, field: String, value: Long): Unit = {
+    this.redisConnection.hincrBy(key, field, value)
+  }
+
+  def hIncByWithRetry(key: String, field: String, value: Long): Unit = {
+    try {
+      hIncBy(key, field, value)
+    } catch {
+      case ex: JedisException => {
+        logger.error(s"Exception while incrementing count key=${key}, field=${field}, value=${value}", ex)
+        this.redisConnection.close()
+        this.redisConnection = redisConnect.getConnection(dbIndex)
+        hIncBy(key, field, value)
+      }
+    }
+  }
+
 }
 
 // $COVERAGE-ON$
\ No newline at end of file
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala
index cbd9586bfa..357ebb0206 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/Events.scala
@@ -4,6 +4,7 @@ import java.util
 
 import com.google.gson.Gson
 import org.sunbird.dp.core.reader.Telemetry
+import org.sunbird.dp.core.util.JSONUtil
 
 abstract class Events(val map: util.Map[String, Any]) {
 
@@ -23,7 +24,8 @@ abstract class Events(val map: util.Map[String, Any]) {
 
   def getMap(): util.Map[String, Any] = telemetry.getMap
 
-  def getJson(): String = new Gson().toJson(getMap)
+  // def getJson(): String = new Gson().toJson(getMap())
+  def getJson(): String = JSONUtil.serialize(getMap())
 
   def mid(): String = telemetry.read[String](keyPath = EventsPath.MID_PATH).orNull
 
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala
index a019234170..b6b5da695b 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/domain/EventsPath.scala
@@ -48,4 +48,6 @@ object EventsPath {
   val OBJECT_ROLLUP_L1 = s"$OBJECT_PATH.rollup.l1"
   val SYNC_TS_PATH ="syncts"
   val TAGS_PATH ="tags"
+  val OBJECT_ROLLUP_L2 = s"$OBJECT_PATH.rollup.l2"
+  val L2_DATA_PATH = "l2data"
 }
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala
index a7705de962..f15049a5ef 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseJobConfig.scala
@@ -15,9 +15,13 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl
 
   implicit val metricTypeInfo: TypeInformation[String] = TypeExtractor.getForClass(classOf[String])
 
-  val kafkaBrokerServers: String = config.getString("kafka.broker-servers")
-  val zookeeper: String = config.getString("kafka.zookeeper")
+  val kafkaProducerBrokerServers: String = config.getString("kafka.producer.broker-servers")
+  val kafkaConsumerBrokerServers: String = config.getString("kafka.consumer.broker-servers")
+  // Producer Properties
   val kafkaProducerMaxRequestSize: Int = config.getInt("kafka.producer.max-request-size")
+  val kafkaProducerBatchSize: Int = config.getInt("kafka.producer.batch.size")
+  val kafkaProducerLingerMs: Int = config.getInt("kafka.producer.linger.ms")
+  val kafkaProducerCompression: String = if (config.hasPath("kafka.producer.compression")) config.getString("kafka.producer.compression") else "snappy"
   val groupId: String = config.getString("kafka.groupId")
   val restartAttempts: Int = config.getInt("task.restart-strategy.attempts")
   val delayBetweenAttempts: Long = config.getLong("task.restart-strategy.delay")
@@ -35,14 +39,16 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl
   val metaRedisPort: Int = Option(config.getInt("redis-meta.port")).getOrElse(6379)
 
   // Checkpointing config
+  val enableCompressedCheckpointing: Boolean = config.getBoolean("task.checkpointing.compressed")
   val checkpointingInterval: Int = config.getInt("task.checkpointing.interval")
+  val checkpointingPauseSeconds: Int = config.getInt("task.checkpointing.pause.between.seconds")
   val enableDistributedCheckpointing: Option[Boolean] = if (config.hasPath("job")) Option(config.getBoolean("job.enable.distributed.checkpointing")) else None
   val checkpointingBaseUrl: Option[String] = if (config.hasPath("job")) Option(config.getString("job.statebackend.base.url")) else None
 
 
   def kafkaConsumerProperties: Properties = {
     val properties = new Properties()
-    properties.setProperty("bootstrap.servers", kafkaBrokerServers)
+    properties.setProperty("bootstrap.servers", kafkaConsumerBrokerServers)
     properties.setProperty("group.id", groupId)
     properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed")
     kafkaAutoOffsetReset.map { properties.setProperty("auto.offset.reset", _) }
@@ -51,10 +57,10 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl
 
   def kafkaProducerProperties: Properties = {
     val properties = new Properties()
-    properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaBrokerServers)
-    properties.put(ProducerConfig.LINGER_MS_CONFIG, new Integer(10))
-    properties.put(ProducerConfig.BATCH_SIZE_CONFIG, new Integer(16384 * 4))
-    properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy")
+    properties.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, kafkaProducerBrokerServers)
+    properties.put(ProducerConfig.LINGER_MS_CONFIG, new Integer(kafkaProducerLingerMs))
+    properties.put(ProducerConfig.BATCH_SIZE_CONFIG, new Integer(kafkaProducerBatchSize))
+    properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, kafkaProducerCompression)
     properties.put(ProducerConfig.MAX_REQUEST_SIZE_CONFIG, new Integer(kafkaProducerMaxRequestSize))
     properties
   }
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala
index a05b5b585b..3ffc043d28 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/BaseProcessFunction.scala
@@ -1,11 +1,14 @@
 package org.sunbird.dp.core.job
 
+import java.lang
 import java.util.concurrent.ConcurrentHashMap
 import java.util.concurrent.atomic.AtomicLong
 
 import org.apache.flink.api.scala.metrics.ScalaGauge
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.apache.flink.streaming.api.functions.windowing.ProcessWindowFunction
+import org.apache.flink.streaming.api.windowing.windows.GlobalWindow
 import org.apache.flink.util.Collector
 
 case class Metrics(metrics: ConcurrentHashMap[String, AtomicLong]) {
@@ -44,3 +47,26 @@ abstract class BaseProcessFunction[T, R](config: BaseJobConfig) extends ProcessF
     processElement(event, context, metrics)
   }
 }
+
+abstract class WindowBaseProcessFunction[I, O, K](config: BaseJobConfig) extends ProcessWindowFunction[I, O, K, GlobalWindow] with BaseDeduplication with JobMetrics {
+
+  private val metrics: Metrics = registerMetrics(metricsList())
+
+  override def open(parameters: Configuration): Unit = {
+    metricsList().map { metric =>
+      getRuntimeContext.getMetricGroup.addGroup(config.jobName)
+        .gauge[Long, ScalaGauge[Long]](metric, ScalaGauge[Long](() => metrics.getAndReset(metric)))
+    }
+  }
+
+  def metricsList(): List[String]
+
+  def process(key: K,
+              context: ProcessWindowFunction[I, O, K, GlobalWindow]#Context,
+              elements: lang.Iterable[I],
+              metrics: Metrics): Unit
+
+  override def process(key: K, context: ProcessWindowFunction[I, O, K, GlobalWindow]#Context, elements: lang.Iterable[I], out: Collector[O]): Unit = {
+    process(key, context, elements, metrics)
+  }
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala
index 5845611dc7..87dc28fd8f 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/job/FlinkKafkaConnector.scala
@@ -4,7 +4,7 @@ import java.util
 
 import org.apache.flink.streaming.api.functions.sink.SinkFunction
 import org.apache.flink.streaming.api.functions.source.SourceFunction
-import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer}
+import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer, KafkaDeserializationSchema}
 import org.apache.flink.streaming.connectors.kafka.FlinkKafkaProducer.Semantic
 import org.sunbird.dp.core.domain.Events
 import org.sunbird.dp.core.serde._
@@ -23,6 +23,10 @@ class FlinkKafkaConnector(config: BaseJobConfig) extends Serializable {
     new FlinkKafkaConsumer[String](kafkaTopic, new StringDeserializationSchema, config.kafkaConsumerProperties)
   }
 
+  def kafkaBytesSource(kafkaTopic: String): SourceFunction[Array[Byte]] = {
+    new FlinkKafkaConsumer[Array[Byte]](kafkaTopic, new ByteDeserializationSchema, config.kafkaConsumerProperties)
+  }
+
   def kafkaStringSink(kafkaTopic: String): SinkFunction[String] = {
     new FlinkKafkaProducer[String](kafkaTopic, new StringSerializationSchema(kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE)
   }
@@ -36,4 +40,7 @@ class FlinkKafkaConnector(config: BaseJobConfig) extends Serializable {
       new EventSerializationSchema[T](kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE)
   }
 
+  def kafkaBytesSink(kafkaTopic: String): SinkFunction[Array[Byte]] = {
+    new FlinkKafkaProducer[Array[Byte]](kafkaTopic, new ByteSerializationSchema(kafkaTopic), config.kafkaProducerProperties, Semantic.AT_LEAST_ONCE)
+  }
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala
new file mode 100644
index 0000000000..8ad7aa838d
--- /dev/null
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/ByteSerde.scala
@@ -0,0 +1,31 @@
+package org.sunbird.dp.core.serde
+
+import java.nio.charset.StandardCharsets
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema}
+import org.apache.kafka.clients.consumer.ConsumerRecord
+import org.apache.kafka.clients.producer.ProducerRecord
+
+class ByteDeserializationSchema extends KafkaDeserializationSchema[Array[Byte]] {
+
+    override def isEndOfStream(nextElement: Array[Byte]): Boolean = false
+
+    override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): Array[Byte] = {
+        record.value()
+    }
+
+    override def getProducedType: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]])
+}
+
+class ByteSerializationSchema(topic: String, key: Option[String] = None) extends KafkaSerializationSchema[Array[Byte]] {
+
+    private val serialVersionUID = -4284080856874185929L
+
+    override def serialize(element: Array[Byte], timestamp: java.lang.Long): ProducerRecord[Array[Byte], Array[Byte]] = {
+        key.map { kafkaKey =>
+            new ProducerRecord[Array[Byte], Array[Byte]](topic, kafkaKey.getBytes(StandardCharsets.UTF_8), element)
+        }.getOrElse(new ProducerRecord[Array[Byte], Array[Byte]](topic, element))
+    }
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala
index 06e840faf0..ebe37c12a3 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/EventSerde.scala
@@ -9,19 +9,29 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema}
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.kafka.clients.producer.ProducerRecord
+import org.slf4j.LoggerFactory
 import org.sunbird.dp.core.domain.Events
+import org.sunbird.dp.core.util.JSONUtil
 
 import scala.reflect.{ClassTag, classTag}
 
 class EventDeserializationSchema[T <: Events](implicit ct: ClassTag[T]) extends KafkaDeserializationSchema[T] {
   private val serialVersionUID = - 7339003654529835367L
-
   override def isEndOfStream(nextElement: T): Boolean = false
+  private[this] val logger = LoggerFactory.getLogger(classOf[EventDeserializationSchema[Events]])
 
   override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): T = {
-    val parsedString = new String(record.value(), StandardCharsets.UTF_8)
-    val result = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass)
-    ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(result).asInstanceOf[T]
+    // val parsedString = new String(record.value(), StandardCharsets.UTF_8)
+    // val result = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass)
+    try {
+      val result = JSONUtil.deserialize[util.HashMap[String, AnyRef]](record.value())
+      ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(result).asInstanceOf[T]
+    }
+    catch {
+      case ex: Exception =>
+        logger.error("Exception when parsing event from kafka: " + record, ex)
+        ct.runtimeClass.getConstructor(classOf[util.Map[String, AnyRef]]).newInstance(new util.HashMap[String, AnyRef]()).asInstanceOf[T]
+    }
   }
 
   override def getProducedType: TypeInformation[T] = TypeExtractor.getForClass(classTag[T].runtimeClass).asInstanceOf[TypeInformation[T]]
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala
index 7d5fa57386..bfb7a1fa41 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/serde/MapSerde.scala
@@ -9,6 +9,8 @@ import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.apache.flink.streaming.connectors.kafka.{KafkaDeserializationSchema, KafkaSerializationSchema}
 import org.apache.kafka.clients.consumer.ConsumerRecord
 import org.apache.kafka.clients.producer.ProducerRecord
+import org.sunbird.dp.core.util.JSONUtil
+
 import collection.JavaConverters._
 
 class MapDeserializationSchema extends KafkaDeserializationSchema[util.Map[String, AnyRef]] {
@@ -18,9 +20,11 @@ class MapDeserializationSchema extends KafkaDeserializationSchema[util.Map[Strin
   override def isEndOfStream(nextElement: util.Map[String, AnyRef]): Boolean = false
 
   override def deserialize(record: ConsumerRecord[Array[Byte], Array[Byte]]): util.Map[String, AnyRef] = {
-    val parsedString = new String(record.value(), StandardCharsets.UTF_8)
-    val recordMap = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass).asScala
-    recordMap.asJava
+    // val parsedString = new String(record.value(), StandardCharsets.UTF_8)
+    // val recordMap = new Gson().fromJson(parsedString, new util.HashMap[String, AnyRef]().getClass).asScala
+    val recordMap = JSONUtil.deserialize[util.HashMap[String, AnyRef]](record.value())
+    // recordMap.asJava
+    recordMap
   }
 
   override def getProducedType: TypeInformation[util.Map[String, AnyRef]] = TypeExtractor.getForClass(classOf[util.Map[String, AnyRef]])
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala
index 3b046c4d09..87a23ca284 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/CassandraUtil.scala
@@ -4,15 +4,17 @@ import java.util
 
 import com.datastax.driver.core._
 import com.datastax.driver.core.exceptions.DriverException
-import com.datastax.driver.core.querybuilder.Insert
+import org.slf4j.LoggerFactory
 
 class CassandraUtil(host: String, port: Int) {
 
-
+  val logger = LoggerFactory.getLogger("CassandraUtil")
+  val options : QueryOptions = new QueryOptions()
   val cluster = {
     Cluster.builder()
       .addContactPoint(host)
       .withPort(port)
+      .withQueryOptions(options.setConsistencyLevel(ConsistencyLevel.QUORUM))
       .withoutJMXReporting()
       .build()
   }
@@ -29,8 +31,11 @@ class CassandraUtil(host: String, port: Int) {
       rs.all
     } catch {
       case ex: DriverException =>
-        this.reconnect()
-        this.find(query)
+        logger.info(s"Failed cassandra query is ${query}")
+        ex.printStackTrace()
+        throw ex
+        // this.reconnect()
+        // this.find(query)
     }
   }
 
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala
index becfb01166..6bbd184446 100644
--- a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/FlinkUtil.scala
@@ -13,18 +13,19 @@ object FlinkUtil {
 
   def getExecutionContext(config: BaseJobConfig): StreamExecutionEnvironment = {
     val env: StreamExecutionEnvironment = StreamExecutionEnvironment.getExecutionEnvironment
+    env.getConfig.setUseSnapshotCompression(config.enableCompressedCheckpointing)
     env.enableCheckpointing(config.checkpointingInterval)
 
     /**
       * Use Blob storage as distributed state backend if enabled
       */
-
     config.enableDistributedCheckpointing match {
       case Some(true) => {
         val stateBackend: StateBackend = new FsStateBackend(s"${config.checkpointingBaseUrl.getOrElse("")}/${config.jobName}", true)
         env.setStateBackend(stateBackend)
         val checkpointConfig: CheckpointConfig = env.getCheckpointConfig
         checkpointConfig.enableExternalizedCheckpoints(ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION)
+        checkpointConfig.setMinPauseBetweenCheckpoints(config.checkpointingPauseSeconds     )
       }
       case _ => // Do nothing
     }
diff --git a/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala
new file mode 100644
index 0000000000..6453191e80
--- /dev/null
+++ b/data-pipeline-flink/dp-core/src/main/scala/org/sunbird/dp/core/util/JSONUtil.scala
@@ -0,0 +1,46 @@
+package org.sunbird.dp.core.util
+
+import java.lang.reflect.{ParameterizedType, Type}
+
+import com.fasterxml.jackson.annotation.JsonInclude.Include
+import com.fasterxml.jackson.core.JsonGenerator.Feature
+import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
+
+import com.fasterxml.jackson.core.`type`.TypeReference
+
+object JSONUtil {
+
+  @transient val mapper = new ObjectMapper()
+  mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
+  mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false)
+  mapper.configure(Feature.WRITE_BIGDECIMAL_AS_PLAIN, true)
+  mapper.setSerializationInclusion(Include.NON_NULL)
+
+  @throws(classOf[Exception])
+  def serialize(obj: AnyRef): String = {
+    mapper.writeValueAsString(obj)
+  }
+
+  def deserialize[T: Manifest](json: String): T = {
+    mapper.readValue(json, typeReference[T])
+  }
+
+  def deserialize[T: Manifest](json: Array[Byte]): T = {
+    mapper.readValue(json, typeReference[T])
+  }
+
+  private[this] def typeReference[T: Manifest] = new TypeReference[T] {
+    override def getType = typeFromManifest(manifest[T])
+  }
+
+
+  private[this] def typeFromManifest(m: Manifest[_]): Type = {
+    if (m.typeArguments.isEmpty) { m.runtimeClass }
+    else new ParameterizedType {
+      def getRawType = m.runtimeClass
+      def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray
+      def getOwnerType = null
+    }
+  }
+
+}
diff --git a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
index a0c6d2fe87..2c678d1993 100644
--- a/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
+++ b/data-pipeline-flink/dp-core/src/test/resources/base-test.conf
@@ -1,6 +1,6 @@
 kafka {
-  broker-servers = "localhost:9093"
-  zookeeper = "localhost:2183"
+  producer.broker-servers = "localhost:9093"
+  consumer.broker-servers = "localhost:9093"
   map.input.topic = "local.telemetry.map.input"
   map.output.topic = "local.telemetry.map.output"
   event.input.topic = "local.telemetry.event.input"
@@ -12,11 +12,16 @@ kafka {
   auto.offset.reset = "earliest"
   producer {
     max-request-size = 102400
+    batch.size = 8192
+    linger.ms = 1
+    compression = "snappy"
   }
 }
 
 task {
+  checkpointing.compressed = true
   checkpointing.interval = 60000
+  checkpointing.pause.between.seconds = 30000
   restart-strategy.attempts = 1
   restart-strategy.delay = 10000
   parallelism = 1
diff --git a/data-pipeline-flink/dp-core/src/test/resources/test.conf b/data-pipeline-flink/dp-core/src/test/resources/test.conf
index f922afa615..22665e323e 100644
--- a/data-pipeline-flink/dp-core/src/test/resources/test.conf
+++ b/data-pipeline-flink/dp-core/src/test/resources/test.conf
@@ -5,20 +5,25 @@ kafka {
   event.output.topic = "local.telemetry.event.output"
   string.input.topic = "local.telemetry.string.input"
   string.output.topic = "local.telemetry.string.output"
-  broker-servers = "localhost:9093"
-  zookeeper = "localhost:2183"
+  producer.broker-servers = "localhost:9093"
+  consumer.broker-servers = "localhost:9093"
   groupId = "pipeline-preprocessor-group"
   auto.offset.reset = "earliest"
   producer {
     max-request-size = 102400
+    batch.size = 8192
+    linger.ms = 1
+    compression = "snappy"
   }
 }
 
 kafka.output.metrics.topic = "pipeline_metrics"
 task {
+  checkpointing.compressed = true
   parallelism = 1
   consumer.parallelism = 1
   checkpointing.interval = 60000
+  checkpointing.pause.between.seconds = 30000
   metrics.window.size = 100 # 3 min
   restart-strategy.attempts = 1 # retry once
   restart-strategy.delay = 1000 # in milli-seconds
diff --git a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala
index fedcfffea7..32ac8e0006 100644
--- a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala
+++ b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/fixture/EventFixture.scala
@@ -32,19 +32,24 @@ object EventFixture {
       |  event.output.topic = "local.telemetry.event.output"
       |  string.input.topic = "local.telemetry.string.input"
       |  string.output.topic = "local.telemetry.string.output"
-      |  broker-servers = "localhost:9093"
-      |  zookeeper = "localhost:2183"
+      |  producer.broker-servers = "localhost:9093"
+      |  consumer.broker-servers = "localhost:9093"
       |  groupId = "pipeline-preprocessor-group"
       |  auto.offset.reset = "earliest"
       |  producer {
       |     max-request-size = 102400
+      |     batch.size = 8192
+      |     linger.ms = 1
+      |     compression = "snappy"
       |  }
       |}
       |
       |task {
       |  parallelism = 2
       |  consumer.parallelism = 1
+      |  checkpointing.compressed = true
       |  checkpointing.interval = 60000
+      |  checkpointing.pause.between.seconds = 30000
       |  metrics.window.size = 100 # 3 min
       |  restart-strategy.attempts = 1 # retry once
       |  restart-strategy.delay = 1000 # in milli-seconds
diff --git a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala
index d21d177d29..7907b5195e 100644
--- a/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala
+++ b/data-pipeline-flink/dp-core/src/test/scala/org/sunbird/spec/BaseProcessFunctionTestSpec.scala
@@ -14,6 +14,7 @@ import org.apache.kafka.common.serialization.StringDeserializer
 import org.scalatest.Matchers
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.core.util.FlinkUtil
+import scala.concurrent.duration._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -68,6 +69,12 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers {
       |"mid":"LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c","type":"events"}
     """.stripMargin
 
+  val EVENT_INVALID =
+    """
+      |"{\"eid\":\"AUDIT\",\"ets\":1605674597050,\"ver\":\"3.0\",\"mid\":\"LP.1605674597050.e920319b-b6e4-4be8-877d-f4c588ee7a3c\",\"actor\":{\"id\":\"9bb884fc-8a56-4727-9522-25a7d5b8ea06\",\"type\":\"User\"},\"context\":{\"channel\":\"ORG_001\",\"pdata\":{\"pid\":\"lms-service\",\"ver\":\"1.0\"},\"env\":\"CourseBatch\",\"cdata\":[{\"id\":\"do_213153022369169408120\",\"type\":\"Course\"},{\"id\":\"01315302480262758472\",\"type\":\"CourseBatch\"}]},\"object\":{\"id\":\"9bb884fc-8a56-4727-9522-25a7d5b8ea06\",\"type\":\"User\",\"rollup\":{\"l1\":\"do_213153022369169408120\"}},\"edata\":{\"state\":\"Create\",\"type\":\"enrol\",\"props\":[\"courseId\",\"enrolledDate\",\"userId\",\"batchId\",\"active\"]}}"
+      |
+      |""".stripMargin
+
   val customKafkaConsumerProperties: Map[String, String] =
     Map[String, String]("auto.offset.reset" -> "earliest", "group.id" -> "test-event-schema-group")
   implicit val embeddedKafkaConfig: EmbeddedKafkaConfig =
@@ -89,6 +96,7 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers {
     publishStringMessageToKafka(bsConfig.kafkaEventInputTopic, SHARE_EVENT)
     publishStringMessageToKafka(bsConfig.kafkaMapInputTopic, EVENT_WITH_MESSAGE_ID)
     publishStringMessageToKafka(bsConfig.kafkaStringInputTopic, SHARE_EVENT)
+    publishStringMessageToKafka(bsConfig.kafkaEventInputTopic, EVENT_INVALID)
 
     flinkCluster.before()
   }
@@ -139,21 +147,26 @@ class BaseProcessFunctionTestSpec extends BaseSpec with Matchers {
       env.execute("TestSerDeFunctionality")
     }
 
-    val eventSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaEventOutputTopic, 2)
-    val eventSchemaDuplicates = consumeNumberMessagesFrom[String](bsConfig.kafkaEventDuplicateTopic, 1)
-    val mapSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaMapOutputTopic, 1)
-    val stringSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaStringOutputTopic, 1)
-
-    eventSchemaMessages.size should be (2)
-    eventSchemaDuplicates.size should be (1)
-    mapSchemaMessages.size should be (1)
-    stringSchemaMessages.size should be (1)
-
-    retrieveMid(mapSchemaMessages.head) should be ("56c0c430-748b-11e8-ae77-cd19397ca6b0")
-    retrieveMid(eventSchemaDuplicates.head) should be ("02ba33e5-15fe-4ec5-b32")
-    retrieveMid(stringSchemaMessages.head) should be ("02ba33e5-15fe-4ec5-b32")
-    retrieveMid(eventSchemaMessages.head) should be ("02ba33e5-15fe-4ec5-b32")
-    retrieveMid(eventSchemaMessages.last) should be ("LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c")
+    // val test = consumeNumberMessagesFromTopics[String](topics = Set(bsConfig.kafkaEventOutputTopic), number = 2, timeout = 30.seconds).values.toList
+    try {
+      val eventSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaEventOutputTopic, 2)
+      val eventSchemaDuplicates = consumeNumberMessagesFrom[String](bsConfig.kafkaEventDuplicateTopic, 1)
+      val mapSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaMapOutputTopic, 1)
+      val stringSchemaMessages = consumeNumberMessagesFrom[String](bsConfig.kafkaStringOutputTopic, 1)
+
+      eventSchemaMessages.size should be(2)
+      eventSchemaDuplicates.size should be(1)
+      mapSchemaMessages.size should be(1)
+      stringSchemaMessages.size should be(1)
+
+      retrieveMid(mapSchemaMessages.head) should be("56c0c430-748b-11e8-ae77-cd19397ca6b0")
+      retrieveMid(eventSchemaDuplicates.head) should be("02ba33e5-15fe-4ec5-b32")
+      retrieveMid(stringSchemaMessages.head) should be("02ba33e5-15fe-4ec5-b32")
+      retrieveMid(eventSchemaMessages.head) should be("02ba33e5-15fe-4ec5-b32")
+      retrieveMid(eventSchemaMessages.last) should be("LP.1586994119534.4bfe9b31-216d-46ea-8e60-d7ea1b1a103c")
+    } catch {
+        case ex: Exception => println("Error occurred when consuming events from Embedded Kafka...")
+    }
 
   }
 
diff --git a/data-pipeline-flink/druid-events-validator/pom.xml b/data-pipeline-flink/druid-events-validator/pom.xml
index 7bba6120d8..9090cf6588 100644
--- a/data-pipeline-flink/druid-events-validator/pom.xml
+++ b/data-pipeline-flink/druid-events-validator/pom.xml
@@ -31,7 +31,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -61,13 +61,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -80,7 +80,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -108,7 +108,14 @@
         <sourceDirectory>src/main/scala</sourceDirectory>
         <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
-
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -156,10 +163,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf b/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf
index 66cf82cd5e..71ccc4b25e 100644
--- a/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf
+++ b/data-pipeline-flink/druid-events-validator/src/main/resources/druid-events-validator.conf
@@ -11,8 +11,7 @@ kafka {
 
 task {
   consumer.parallelism = 1
-  validator.parallelism = 1
-  router.parallelism = 1
+  downstream.operators.parallelism = 1
   druid.validation.enabled = true
   druid.deduplication.enabled = true
 }
diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties b/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties
deleted file mode 100644
index a15d8c34cb..0000000000
--- a/data-pipeline-flink/druid-events-validator/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=druid-events-validator.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json
index c6f61d886d..19f440abab 100644
--- a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json
+++ b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/filter.json
@@ -2,14 +2,5 @@
   "type": "object",
   "properties": {
   },
-  "anyOf": [
-    {
-      "not": {
-        "required": [
-          "dialCodes"
-        ]
-      }
-    }
-  ],
   "additionalProperties": true
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json
index 73ddce9313..3e27c731b3 100644
--- a/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json
+++ b/data-pipeline-flink/druid-events-validator/src/main/resources/schemas/telemetry/search.json
@@ -53,7 +53,8 @@
               "type": "number"
             },
             "filters": {
-              "$ref": "resource:/schemas/telemetry/filter.json"
+              "id": "http://api.sunbird.org/telemetry/edata/filters",
+              "type": "object"
             },
             "topn": {
               "id": "http://api.sunbird.org/telemetry/edata/topn",
diff --git a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala
index afbeea61b8..97ba43782d 100644
--- a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala
+++ b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorConfig.scala
@@ -16,8 +16,7 @@ class DruidValidatorConfig(override val config: Config) extends BaseJobConfig(co
   val cacheExpirySeconds: Int = config.getInt("redis.database.key.expiry.seconds")
 
   override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
-  val validatorParallelism: Int = config.getInt("task.validator.parallelism")
-  val routerParallelism: Int = config.getInt("task.router.parallelism")
+  val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism")
 
   val druidValidationEnabled: Boolean = config.getBoolean("task.druid.validation.enabled")
   val druidDeduplicationEnabled: Boolean = config.getBoolean("task.druid.deduplication.enabled")
diff --git a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala
index 01c273e099..77fc43c8fe 100644
--- a/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala
+++ b/data-pipeline-flink/druid-events-validator/src/main/scala/org/sunbird/dp/validator/task/DruidValidatorStreamTask.scala
@@ -49,22 +49,28 @@ class DruidValidatorStreamTask(config: DruidValidatorConfig, kafkaConnector: Fli
       .uid(config.druidValidatorConsumer).setParallelism(config.kafkaConsumerParallelism)
       .rebalance()
       .process(new DruidValidatorFunction(config)).name(config.druidValidatorFunction).uid(config.druidValidatorFunction)
-      .setParallelism(config.validatorParallelism)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
+    validationDataStream.getSideOutput(config.telemetryRouterOutputTag)
 
     /**
      * Separate sinks for valid telemetry events, valid summary events, valid error events, valid log events and invalid events
      */
     validationDataStream.getSideOutput(config.telemetryRouterOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaTelemetryRouteTopic))
       .name(config.telemetryEventsProducer).uid(config.telemetryEventsProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     validationDataStream.getSideOutput(config.summaryRouterOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaSummaryRouteTopic))
       .name(config.summaryEventsProducer).uid(config.summaryEventsProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     validationDataStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic))
       .name(config.druidDuplicateEventsProducer).uid(config.druidDuplicateEventsProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     validationDataStream.getSideOutput(config.invalidEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaFailedTopic))
       .name(config.druidInvalidEventsProducer).uid(config.druidInvalidEventsProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     env.execute(config.jobName)
 
diff --git a/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf b/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf
index 4ecbc2c0b6..f1baad4e8c 100644
--- a/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf
+++ b/data-pipeline-flink/druid-events-validator/src/test/resources/test.conf
@@ -10,8 +10,7 @@ kafka {
 }
 
 task {
-  validator.parallelism = 1
-  router.parallelism = 1
+  downstream.operators.parallelism = 1
   druid.validation.enabled = true
   druid.deduplication.enabled = true
 }
diff --git a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index ab2177a0d5..791fccd848 100644
--- a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -81,7 +81,7 @@ object EventFixture {
       |"objecttype":"Content","mediatype":"content","status":"Live"}}
     """.stripMargin
 
-  val VALID_SERACH_EVENT: String =
+  val VALID_SEARCH_EVENT: String =
     """
       |{"eid":"SEARCH","ver":"3.0","syncts":1.59518415538E12,"ets":1.59518415538E12,"flags":
       |{"pp_validation_processed":true,"pp_duplicate":false,"device_denorm":false,"dialcode_denorm":true,
@@ -97,4 +97,21 @@ object EventFixture {
       |"pid":"search-service"},"did":"79838ccb0ff2c7d0a9dd05f5b337fbca","env":"search","channel":"ROOT_ORG"},
       |"@version":"1","object":{"id":"KLQ2G7","type":"DialCode"}}
     """.stripMargin
+
+  val SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY: String =
+    """
+      |{"eid":"SEARCH","ver":"3.0","syncts":1.59518415538E12,"ets":1.59518415538E12,"flags":
+      |{"pp_validation_processed":true,"pp_duplicate":false,"device_denorm":false,"dialcode_denorm":true,
+      |"content_denorm":false},"dialcodedata":{"identifier":"KLQ2G7","channel":"0123221617357783046602",
+      |"publisher":"MHPUBLISHER","status":2.0},"mid":"invalid_dialcode_key",
+      |"type":"events","tags":["kp-events"],"actor":{"id":"org.sunbird.learning.platform","type":"System"},
+      |"edata":{"topn":[{"identifier":"do_312528046917705728246886"}],"query":"","size":7.0,"type":"content",
+      |"filters":{"contentType":["TextBookUnit","Resource","TextBook","Collection","Course"],"mimeType":{},
+      |"resourceType":{},"status":["Live"],"objectType":["Content"],"dialCodes":"KLQ2G7","framework":{},
+      |"compatibilityLevel":{"max":4.0,"min":1.0},"channel":{"ne":["0124433024890224640","0124446042259128320",
+      |"0124487522476933120","0125840271570288640","0124453662635048969"]}},"sort":{}},
+      |"@timestamp":"2020-07-19T18:42:41.524Z","context":{"pdata":{"ver":"1.0","id":"prod.diksha.portal",
+      |"pid":"search-service"},"did":"79838ccb0ff2c7d0a9dd05f5b337fbca","env":"search","channel":"ROOT_ORG"},
+      |"@version":"1","object":{"id":"KLQ2G7","type":"DialCode"}}
+    """.stripMargin
 }
diff --git a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala
index 670971566e..e19508da2e 100644
--- a/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala
+++ b/data-pipeline-flink/druid-events-validator/src/test/scala/org/sunbird/dp/spec/DruidValidatorStreamTaskTestSpec.scala
@@ -13,6 +13,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceCont
 import org.apache.flink.test.util.MiniClusterWithClientResource
 import org.mockito.Mockito
 import org.mockito.Mockito.when
+import org.sunbird.dp.core.domain.EventsPath
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.fixture.EventFixture
 import org.sunbird.dp.validator.domain.Event
@@ -20,6 +21,8 @@ import org.sunbird.dp.validator.task.{DruidValidatorConfig, DruidValidatorStream
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
 import redis.embedded.RedisServer
 
+import scala.collection.JavaConverters._
+
 
 class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
 
@@ -32,8 +35,6 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
       .build)
 
     var redisServer: RedisServer = _
-    // val config: Config = ConfigFactory.load("test.conf")
-    // val druidValidatorConfig: DruidValidatorConfig = new DruidValidatorConfig(config)
     var config: Config = _
     var druidValidatorConfig: DruidValidatorConfig = _
     val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
@@ -45,16 +46,6 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
         redisServer.start()
 
         BaseMetricsReporter.gaugeMetrics.clear()
-
-        /*
-        when(mockKafkaUtil.kafkaEventSource[Event](druidValidatorConfig.kafkaInputTopic)).thenReturn(new DruidValidatorEventSource)
-
-        when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaDuplicateTopic)).thenReturn(new DupEventsSink)
-        when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaTelemetryRouteTopic)).thenReturn(new TelemetryEventsSink)
-        when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaSummaryRouteTopic)).thenReturn(new SummaryEventsSink)
-        when(mockKafkaUtil.kafkaEventSink[Event](druidValidatorConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink)
-        */
-
         flinkCluster.before()
     }
 
@@ -81,24 +72,27 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
 
         val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil)
         task.process()
-        TelemetryEventsSink.values.size() should be (2)
+        TelemetryEventsSink.values.size() should be (3)
         SummaryEventsSink.values.size() should be (1)
         FailedEventsSink.values.size() should be (1)
         DupEventsSink.values.size() should be (1)
 
+        val invalidDialCodeKeyEvent = TelemetryEventsSink.values.asScala.filter(event => event.mid() === "invalid_dialcode_key").head
+        invalidDialCodeKeyEvent.getTelemetry.read(s"${EventsPath.EDTA_FILTERS}.dialCodes") should not be None
+
         DupEventsSink.values.get(0).getFlags.get("dv_duplicate").booleanValue() should be(true)
         
         FailedEventsSink.values.get(0).getFlags.get("dv_processed").booleanValue() should be(false)
         FailedEventsSink.values.get(0).getFlags.get("dv_validation_failed").booleanValue() should be(true)
         
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (3)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (4)
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (1)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (4)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (5)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (2)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3)
 
     }
 
@@ -122,7 +116,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
 
         val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil)
         task.process()
-        TelemetryEventsSink.values.size() should be (4)
+        TelemetryEventsSink.values.size() should be (5)
         SummaryEventsSink.values.size() should be (1)
         FailedEventsSink.values.size() should be (0)
         DupEventsSink.values.size() should be (0)
@@ -134,7 +128,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (0)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (5)
 
     }
 
@@ -158,19 +152,22 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
 
         val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil)
         task.process()
-        TelemetryEventsSink.values.size() should be (3)
+        TelemetryEventsSink.values.size() should be (4)
         SummaryEventsSink.values.size() should be (1)
         FailedEventsSink.values.size() should be (1)
         DupEventsSink.values.size() should be (0)
 
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (4)
+        val invalidDialCodeKeyEvent = TelemetryEventsSink.values.asScala.filter(event => event.mid() === "invalid_dialcode_key").head
+        invalidDialCodeKeyEvent.getTelemetry.read(s"${EventsPath.EDTA_FILTERS}.dialCodes") should not be None
+
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationSuccessMetricsCount}").getValue() should be (5)
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (1)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (0)
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (0)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4)
 
     }
 
@@ -194,7 +191,7 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
 
         val task = new DruidValidatorStreamTask(druidValidatorConfig, mockKafkaUtil)
         task.process()
-        TelemetryEventsSink.values.size() should be (3)
+        TelemetryEventsSink.values.size() should be (4)
         SummaryEventsSink.values.size() should be (1)
         FailedEventsSink.values.size() should be (0)
         DupEventsSink.values.size() should be (1)
@@ -203,10 +200,10 @@ class DruidValidatorStreamTaskTestSpec extends BaseTestSpec {
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.validationFailureMetricsCount}").getValue() should be (0)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.duplicate-event-count").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (4)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.unique-event-count").getValue() should be (5)
 
         BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.summaryRouterMetricCount}").getValue() should be (1)
-        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (3)
+        BaseMetricsReporter.gaugeMetrics(s"${druidValidatorConfig.jobName}.${druidValidatorConfig.telemetryRouterMetricCount}").getValue() should be (4)
 
     }
 
@@ -219,11 +216,13 @@ class DruidValidatorEventSource  extends SourceFunction[Event] {
         val event1 = gson.fromJson(EventFixture.VALID_DENORM_TELEMETRY_EVENT, new util.LinkedHashMap[String, Any]().getClass)
         val event2 = gson.fromJson(EventFixture.INVALID_DENORM_TELEMETRY_EVENT, new util.LinkedHashMap[String, Any]().getClass)
         val event3 = gson.fromJson(EventFixture.VALID_DENORM_SUMMARY_EVENT, new util.LinkedHashMap[String, Any]().getClass)
-        val event4 = gson.fromJson(EventFixture.VALID_SERACH_EVENT, new util.LinkedHashMap[String, Any]().getClass)
+        val event4 = gson.fromJson(EventFixture.VALID_SEARCH_EVENT, new util.LinkedHashMap[String, Any]().getClass)
+        val event5 = gson.fromJson(EventFixture.SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY, new util.LinkedHashMap[String, Any]().getClass)
         ctx.collect(new Event(event1))
         ctx.collect(new Event(event2))
         ctx.collect(new Event(event3))
         ctx.collect(new Event(event4))
+        ctx.collect(new Event(event5))
         ctx.collect(new Event(event1))
     }
 
diff --git a/data-pipeline-flink/ingest-router/pom.xml b/data-pipeline-flink/ingest-router/pom.xml
new file mode 100644
index 0000000000..103ddd5e14
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/pom.xml
@@ -0,0 +1,217 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <parent>
+        <groupId>org.sunbird.dp</groupId>
+        <artifactId>pipeline-jobs</artifactId>
+        <version>1.0</version>
+    </parent>
+
+    <groupId>org.sunbird.dp.jobs</groupId>
+    <artifactId>ingest-router</artifactId>
+    <version>1.0.0</version>
+    <packaging>jar</packaging>
+    <name>IngestRouter</name>
+    <description>
+        Ingest router
+    </description>
+    
+    <properties>
+		<encoding>UTF-8</encoding>
+		<scoverage.plugin.version>1.4.0</scoverage.plugin.version>
+	</properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.sunbird.dp</groupId>
+            <artifactId>dp-core</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.sunbird.dp</groupId>
+            <artifactId>dp-core</artifactId>
+            <version>1.0.0</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-test-utils_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-runtime_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+            <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+            <groupId>it.ozimov</groupId>
+            <artifactId>embedded-redis</artifactId>
+            <version>0.7.1</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+            <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.12</artifactId>
+            <version>3.0.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.3.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fiftyonred</groupId>
+            <artifactId>mock-jedis</artifactId>
+            <version>0.4.0</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+    		<sourceDirectory>src/main/scala</sourceDirectory>
+		    <testSourceDirectory>src/test/scala</testSourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <!-- Run shade goal on package phase -->
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <artifactSet>
+                                <excludes>
+                                    <exclude>com.google.code.findbugs:jsr305</exclude>
+                                </excludes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <!-- Do not copy the signatures in the META-INF folder.
+                                    Otherwise, this might cause SecurityExceptions when using the JAR. -->
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.sunbird.dp.ingestrouter.task.IngestRouterStreamTask</mainClass>
+                                </transformer>
+                                <!-- append default configs -->
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>reference.conf</resource>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.4.0</version>
+                <configuration>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                    <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.22.2</version>
+                <configuration>
+                    <skipTests>true</skipTests>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest-maven-plugin</artifactId>
+                <version>1.0</version>
+                <configuration>
+                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+                    <junitxml>.</junitxml>
+                    <filereports>dp-duplication-testsuite.txt</filereports>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>test</id>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+			<plugin>
+				<groupId>org.scoverage</groupId>
+				<artifactId>scoverage-maven-plugin</artifactId>
+				<version>${scoverage.plugin.version}</version>
+				<configuration>
+					<scalaVersion>${scala.version}</scalaVersion>
+					<aggregate>true</aggregate>
+					<highlighting>true</highlighting>
+				</configuration>
+			</plugin>
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file
diff --git a/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf b/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf
new file mode 100644
index 0000000000..05580b822a
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/main/resources/ingest-router.conf
@@ -0,0 +1,12 @@
+include "base-config.conf"
+
+kafka {
+  input.topic = ${job.env}".telemetry.ingestion"
+  output.success.topic = ${job.env}".telemetry.ingest"
+  groupId = ${job.env}"-ingest-router-group"
+}
+
+task {
+  consumer.parallelism = 1
+  downstream.operators.parallelism = 1
+}
diff --git a/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala
new file mode 100644
index 0000000000..ce185a0c12
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterConfig.scala
@@ -0,0 +1,48 @@
+package org.sunbird.dp.ingestrouter.task
+
+import java.util
+
+import com.typesafe.config.Config
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.streaming.api.scala.OutputTag
+import org.sunbird.dp.core.job.BaseJobConfig
+
+  class IngestRouterConfig(override val config: Config) extends BaseJobConfig(config, "IngestRouterJob") {
+
+    private val serialVersionUID = 2905979434303791379L
+
+    implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]])
+
+    // Kafka Topics Configuration
+    val kafkaIngestInputTopic: String = config.getString("kafka.ingest.input.topic")
+    val kafkaIngestSuccessTopic: String = config.getString("kafka.ingest.output.success.topic")
+
+    val kafkaRawInputTopic: String = config.getString("kafka.raw.input.topic")
+    val kafkaRawSuccessTopic: String = config.getString("kafka.raw.output.success.topic")
+
+
+    override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
+    val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism")
+    val rawKafkaConsumerParallelism: Int = config.getInt("task.raw.consumer.parallelism")
+    val rawDownstreamOperatorsParallelism: Int = config.getInt("task.raw.downstream.operators.parallelism")
+
+    val EVENTS_OUTPUT_TAG = "events"
+
+    // Metric List
+    val ingestSuccessEventCount = "ingest-success-event-count"
+    val rawSuccessEventCount = "raw-success-event-count"
+
+    val eventsOutputTag: OutputTag[Array[Byte]] = OutputTag[Array[Byte]](EVENTS_OUTPUT_TAG)
+
+    // Consumers
+    val telemetryIngestProcessorConsumer = "ingest-router-consumer"
+    val telemetryRawProcessorConsumer = "raw-router-consumer"
+
+    // Producers
+    val telemetryIngestProcessorProducer = "ingest-router-sink"
+    val telemetryRawProcessorProducer = "raw-router-sink"
+
+
+
+  }
diff --git a/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala
new file mode 100644
index 0000000000..1d61d98f91
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/main/scala/org/sunbird/dp/ingestrouter/task/IngestRouterStreamTask.scala
@@ -0,0 +1,55 @@
+package org.sunbird.dp.ingestrouter.task
+
+import java.io.File
+import java.util
+
+import com.typesafe.config.ConfigFactory
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.java.utils.ParameterTool
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.core.util.FlinkUtil
+
+class IngestRouterStreamTask(config: IngestRouterConfig, kafkaConnector: FlinkKafkaConnector) {
+
+  private val serialVersionUID = -7729362727131516112L
+
+  def process(): Unit = {
+    implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config)
+    implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]])
+
+    val ingestStream =
+      env.addSource(kafkaConnector.kafkaBytesSource(config.kafkaIngestInputTopic), config.telemetryIngestProcessorConsumer)
+        .uid(config.telemetryIngestProcessorConsumer).setParallelism(config.kafkaConsumerParallelism)
+        .addSink(kafkaConnector.kafkaBytesSink(config.kafkaIngestSuccessTopic))
+        .name(config.telemetryIngestProcessorProducer).uid(config.telemetryIngestProcessorProducer)
+        .setParallelism(config.downstreamOperatorsParallelism)
+    val rawStream =
+      env.addSource(kafkaConnector.kafkaBytesSource(config.kafkaRawInputTopic), config.telemetryRawProcessorConsumer)
+        .uid(config.telemetryRawProcessorConsumer).setParallelism(config.rawKafkaConsumerParallelism)
+        .addSink(kafkaConnector.kafkaBytesSink(config.kafkaRawSuccessTopic))
+        .name(config.telemetryRawProcessorProducer).uid(config.telemetryRawProcessorProducer)
+        .setParallelism(config.rawDownstreamOperatorsParallelism)
+
+
+    env.execute(config.jobName)
+  }
+}
+
+// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster
+object IngestRouterStreamTask {
+
+  def main(args: Array[String]): Unit = {
+    val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path"))
+    val config = configFilePath.map {
+      path => ConfigFactory.parseFile(new File(path)).resolve()
+    }.getOrElse(ConfigFactory.load("ingest-router.conf").withFallback(ConfigFactory.systemEnvironment()))
+    val telemetryProcessorConfig = new IngestRouterConfig(config)
+    val kafkaUtil = new FlinkKafkaConnector(telemetryProcessorConfig)
+    val task = new IngestRouterStreamTask(telemetryProcessorConfig, kafkaUtil)
+    task.process()
+  }
+}
+
+// $COVERAGE-ON$
diff --git a/data-pipeline-flink/ingest-router/src/test/resources/test.conf b/data-pipeline-flink/ingest-router/src/test/resources/test.conf
new file mode 100644
index 0000000000..319f0131ce
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/test/resources/test.conf
@@ -0,0 +1,23 @@
+include "base-test.conf"
+
+kafka {
+
+  groupId = "flink-ingest-router-group"
+  ingest {
+    input.topic = "flink.telemetry.ingestion"
+    output.success.topic = "flink.telemetry.ingest"
+  }
+  raw {
+    input.topic = "flink.telemetry.raw.event"
+    output.success.topic = "flink.telemetry.raw"
+  }
+}
+
+task {
+  consumer.parallelism = 1
+  downstream.operators.parallelism = 1
+  raw {
+    consumer.parallelism = 1
+    downstream.operators.parallelism = 1
+  }
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
new file mode 100644
index 0000000000..daab76d0d5
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -0,0 +1,54 @@
+package org.sunbird.dp.fixture
+
+object EventFixture {
+
+  val EVENT_WITH_MESSAGE_ID: String =
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |""".stripMargin
+
+  val EVENT_WITHOUT_MESSAGE_ID: String =
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test1","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test2","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test3","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |""".stripMargin
+
+  val EMPTY_BATCH_EVENTS: String =
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |""".stripMargin
+
+  val UNDEFINED_EVENTS_IN_BATCH: String =
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |""".stripMargin
+
+  val INVALID_BATCH_EVENT: String =
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":["eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |""".stripMargin
+
+  val MISSING_FIELDS_BATCH:String = {
+    """
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"mid":"5734970","channel":"98347593475834","params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","key":"","did":"0958743690"},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}]}
+      |""".stripMargin
+  }
+
+  val RAW_LOG_EVENT:String = {
+    """
+      |{"object":{"rollup":{},"ver":"","type":"content","id":"do_11307732367521382418684"},"ets":1596373063462,"ver":"3.0","eid":"LOG","actor":{"type":"service","id":"1"},"context":{"rollup":{},"sid":"","pdata":{"id":"local.dock.content-service","ver":"1.0","pid":"sunbird-content-service"},"env":"contentplayer","did":"","channel":"sunbird","cdata":[]},"mid":"LOG:89da1a341096f1e5e24806a0250c5c83","tags":[],"edata":{"params":[{"rid":"c1457c60-d4bf-11ea-9313-0912071b8abe"},{},{},{"url":"health"},{"method":"GET"}],"message":"successful","type":"api_access","level":"INFO"},"syncts":1596375302571,"@timestamp":"2020-08-02T13:35:02.571Z","flags":{"ex_processed":true}}
+      |""".stripMargin
+  }
+
+  val RAW_SEARCH_EVENT:String = {
+    """
+      |{"@timestamp":"2020-08-11T10:21:19.464Z","ets":1596001408756,"ver":"3.0","eid":"SEARCH","mid":"LP.1596001408756.045c0aa6-4b70-45d3-b43f-bc908e512175","actor":{"type":"System","id":"org.sunbird.learning.platform"},"context":{"channel":"in.ekstep","pdata":{"pid":"search-service","ver":"1.0","id":"sunbird.Sunbird_Dev.learning.platform"},"env":"search"},"syncts":1596001408756,"@version":"1","edata":{"filters":{"objectType":["Content"],"resourceType":{},"contentType":["course"],"framework":{},"channel":{},"mimeType":{},"status":"Live","identifier":["KP_FT_1564394114456","do_112476372676763648176","do_1124785353783377921154","do_112481468925796352148","do_1124834956377456641118","do_1124835868462612481134","do_1125083286221291521153","do_1125098016170639361238","do_1125110672521134081296","do_11269260687205990411105","do_112717173739429888132","do_112728688963133440110506","do_112736824142061568168","do_11283193441064550414","do_11307319880400076811160","do_2123251055368765441634"]},"query":"","size":14,"sort":{},"topn":[{"identifier":"do_112736824142061568168"},{"identifier":"do_1125110672521134081296"},{"identifier":"do_11269260687205990411105"},{"identifier":"do_112728688963133440110506"},{"identifier":"do_112717173739429888132"}],"type":"content"}}
+      |""".stripMargin
+  }
+
+  val RAW_ERROR_EVENT:String = {
+    """
+      |{"@timestamp":"2020-08-11T10:21:18.518Z","ets":1596483300965,"ver":"3.0","eid":"ERROR","mid":"LP.1596483300965.28e663f6-2994-442a-8f4a-eb1459c17a99","actor":{"type":"User","id":"org.ekstep.learning.platform"},"context":{"channel":"in.ekstep","pdata":{"pid":"learning-service","ver":"1.0","id":"dev.sunbird.learning.platform"},"env":"system"},"syncts":1596483300965,"@version":"1","edata":{"errtype":"system","err":"RESOURCE_NOT_FOUND","stacktrace":"RESOURCE_NOT_FOUND: Resource not found : KP_FT_1596483300820.img\n\tat org.ekstep.learning.actor.ContentStoreActor.onReceive(ContentStoreActor.java:77)\n\tat akka.actor.UntypedActor$$anonfun$receive$1.applyOrElse(UntypedActor.scala:167)\n\tat akka.actor.Actor$class.aroundReceive(Actor.scala:539)\n\tat akka.actor.UntypedActor.aroundReceive(UntypedActor.scala:97)\n\tat akka.actor.ActorCell.receiveMessage(ActorCell.scala:610)\n\tat akka.actor.ActorCell.invoke(ActorCell.scala:579)\n\tat akka.dispatch.Mailbox.processMailbox(Mailbox.scala:268)\n\tat akka.dispatch.Mailbox.run(Mailbox.scala:229)\n\tat akka.dispatch.Mailbox.exec(Mailbox.scala:241)\n\tat akka.dispatch.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)\n\tat akka.dispatch.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)\n\tat akka.dispatch.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)\n\tat akka.dispatch.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)\n"}}
+      |""".stripMargin
+  }
+
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala
new file mode 100644
index 0000000000..0da773599b
--- /dev/null
+++ b/data-pipeline-flink/ingest-router/src/test/scala/org/sunbird/dp/spec/IngestRouterStreamTaskTestSpec.scala
@@ -0,0 +1,131 @@
+package org.sunbird.dp.spec
+
+import java.nio.charset.StandardCharsets
+import java.util
+
+import com.google.gson.Gson
+import com.typesafe.config.Config
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.mockito.Mockito
+import org.mockito.Mockito._
+import org.sunbird.dp.fixture.EventFixture
+import redis.embedded.RedisServer
+import org.apache.flink.streaming.api.functions.sink.SinkFunction
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
+import org.apache.flink.test.util.MiniClusterWithClientResource
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+import com.typesafe.config.ConfigFactory
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.core.domain.Events
+import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.ingestrouter.task.{IngestRouterConfig, IngestRouterStreamTask}
+import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
+
+import collection.JavaConverters._
+
+class IngestRouterStreamTaskTestSpec extends BaseTestSpec {
+
+  implicit val bytesTypeInfo: TypeInformation[Array[Byte]] = TypeExtractor.getForClass(classOf[Array[Byte]])
+
+  val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
+    .setConfiguration(testConfiguration())
+    .setNumberSlotsPerTaskManager(1)
+    .setNumberTaskManagers(1)
+    .build)
+
+  val config: Config = ConfigFactory.load("test.conf")
+  val ingestRouterConfig: IngestRouterConfig = new IngestRouterConfig(config)
+  val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    BaseMetricsReporter.gaugeMetrics.clear()
+
+    when(mockKafkaUtil.kafkaBytesSource(ingestRouterConfig.kafkaIngestInputTopic)).thenReturn(new IngestRouterEventSource)
+    when(mockKafkaUtil.kafkaBytesSink(ingestRouterConfig.kafkaIngestSuccessTopic)).thenReturn(new IngestSuccessEventsSink)
+
+    when(mockKafkaUtil.kafkaBytesSource(ingestRouterConfig.kafkaRawInputTopic)).thenReturn(new RawRouterEventSource)
+    when(mockKafkaUtil.kafkaBytesSink(ingestRouterConfig.kafkaRawSuccessTopic)).thenReturn(new RawSuccessEventsSink)
+
+    flinkCluster.before()
+  }
+
+  override protected def afterAll(): Unit = {
+    super.afterAll()
+    flinkCluster.after()
+  }
+
+  "Ingest router job pipeline" should "just pass through ingest events" in {
+
+      val task = new IngestRouterStreamTask(ingestRouterConfig, mockKafkaUtil)
+      task.process()
+
+      IngestSuccessEventsSink.values.size() should be(3)
+
+      RawSuccessEventsSink.values.size() should be(3)
+
+  }
+
+}
+
+class IngestRouterEventSource extends SourceFunction[Array[Byte]] {
+
+  override def run(ctx: SourceContext[Array[Byte]]) {
+    val gson = new Gson()
+    val event1 = EventFixture.EVENT_WITH_MESSAGE_ID
+    val event2 = EventFixture.EVENT_WITHOUT_MESSAGE_ID
+    val event3 = EventFixture.INVALID_BATCH_EVENT
+    ctx.collect(event1.getBytes(StandardCharsets.UTF_8))
+    ctx.collect(event2.getBytes(StandardCharsets.UTF_8))
+    ctx.collect(event3.getBytes(StandardCharsets.UTF_8))
+  }
+
+  override def cancel() = {}
+
+}
+
+class RawRouterEventSource extends SourceFunction[Array[Byte]] {
+
+  override def run(ctx: SourceContext[Array[Byte]]) {
+    val gson = new Gson()
+    val event1 = EventFixture.RAW_LOG_EVENT
+    val event2 = EventFixture.RAW_SEARCH_EVENT
+    val event3 = EventFixture.RAW_ERROR_EVENT
+    ctx.collect(event1.getBytes(StandardCharsets.UTF_8))
+    ctx.collect(event2.getBytes(StandardCharsets.UTF_8))
+    ctx.collect(event3.getBytes(StandardCharsets.UTF_8))
+
+  }
+
+  override def cancel() = {}
+
+}
+
+
+class IngestSuccessEventsSink extends SinkFunction[Array[Byte]] {
+
+  override def invoke(value: Array[Byte]): Unit = {
+    synchronized {
+      IngestSuccessEventsSink.values.add(value)
+    }
+  }
+}
+
+object IngestSuccessEventsSink {
+  val values: util.List[Array[Byte]] = new util.ArrayList[Array[Byte]]()
+}
+
+class RawSuccessEventsSink extends SinkFunction[Array[Byte]] {
+
+  override def invoke(value: Array[Byte]): Unit = {
+    synchronized {
+      RawSuccessEventsSink.values.add(value)
+    }
+  }
+}
+
+object RawSuccessEventsSink {
+  val values: util.List[Array[Byte]] = new util.ArrayList[Array[Byte]]()
+}
diff --git a/data-pipeline-flink/pipeline-preprocessor/pom.xml b/data-pipeline-flink/pipeline-preprocessor/pom.xml
index d4c09e856e..c346c6f83b 100644
--- a/data-pipeline-flink/pipeline-preprocessor/pom.xml
+++ b/data-pipeline-flink/pipeline-preprocessor/pom.xml
@@ -26,7 +26,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -51,6 +51,11 @@
             <artifactId>commons-lang3</artifactId>
             <version>3.9</version>
         </dependency>
+        <dependency>
+            <groupId>io.github.classgraph</groupId>
+            <artifactId>classgraph</artifactId>
+            <version>4.8.90</version>
+        </dependency>
         <dependency>
             <groupId>org.sunbird.dp</groupId>
             <artifactId>dp-core</artifactId>
@@ -67,13 +72,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -86,7 +91,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -108,7 +113,14 @@
         <sourceDirectory>src/main/scala</sourceDirectory>
         <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
-
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -158,10 +170,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties
deleted file mode 100644
index 77a86a6451..0000000000
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=pipeline-preprocessor.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf
index b4e1550b60..98dd6828c1 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/pipeline-preprocessor.conf
@@ -5,22 +5,23 @@ kafka {
   output.failed.topic = ${job.env}".telemetry.failed"
   output.primary.route.topic = ${job.env}".telemetry.unique"
   output.log.route.topic = ${job.env}".druid.events.log"
-  output.error.route.topic = ${job.env}".druid.events.error"
+  output.error.route.topic = ${job.env}".telemetry.error"
   output.audit.route.topic = ${job.env}".telemetry.audit"
   output.duplicate.topic = ${job.env}".telemetry.duplicate"
+  output.denorm.secondary.route.topic = ${job.env}".telemetry.unique.secondary"
+  output.denorm.primary.route.topic = ${job.env}".telemetry.unique.primary"
   groupId = ${job.env}"-pipeline-preprocessor-group"
 }
 
 task {
   consumer.parallelism = 1
-  telemetry.validation.parallelism = 1
-  telemetry.router.parallelism = 1
-  share.events.flattener.parallelism = 1
+  downstream.operators.parallelism = 1
 }
 
 telemetry.schema.path="schemas/telemetry/3.0"
 default.channel="b00bc992ef25f1a9a8d63291e20efc8d"
-dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"]
+dedup.producer.included.ids = [${job.env}".sunbird.portal", ${job.env}".sunbird.desktop"]
+secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"]
 
 redis {
   database {
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json
index e77e5225c2..a4e4924b87 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/event.json
@@ -46,6 +46,12 @@
     },
     {
       "$ref": "resource:/schemas/telemetry/3.0/exdata.json"
+    },
+    {
+      "$ref": "resource:/schemas/telemetry/3.0/summary.json"
+    },
+    {
+      "$ref": "resource:/schemas/telemetry/3.0/metrics.json"
     }
   ]
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json
new file mode 100644
index 0000000000..db2802812d
--- /dev/null
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/metrics.json
@@ -0,0 +1,63 @@
+{
+  "id": "http://api.ekstep.org/telemetry/metrics",
+  "type": "object",
+  "required": [
+    "eid",
+    "ets",
+    "ver",
+    "mid",
+    "actor",
+    "context",
+    "edata"
+  ],
+  "allOf": [
+    {
+      "$ref": "resource:/schemas/telemetry/3.0/common.json"
+    },
+    {
+      "properties": {
+        "eid": {
+          "id": "http://api.ekstep.org/telemetry/eid",
+          "enum": [
+            "METRICS"
+          ]
+        },
+        "edata": {
+          "id": "http://api.ekstep.org/telemetry/edata",
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "system",
+            "subsystem"
+          ],
+          "properties": {
+            "system": {
+              "id": "http://api.ekstep.org/telemetry/edata/system",
+              "type": "string"
+            },
+            "subsystem": {
+              "id": "http://api.ekstep.org/telemetry/edata/subsystem",
+              "type": "string"
+            },
+            "metrics": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "metric": {
+                    "id": "http://api.ekstep.org/telemetry/edata/metrics/metric",
+                    "type": "string"
+                  },
+                  "value": {
+                    "id": "http://api.ekstep.org/telemetry/edata/metrics/value",
+                    "type": "number"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json
new file mode 100644
index 0000000000..4c2c2b3db5
--- /dev/null
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/resources/schemas/telemetry/3.0/summary.json
@@ -0,0 +1,87 @@
+{
+  "id": "http://api.ekstep.org/telemetry/summary",
+  "type": "object",
+  "required": [
+    "eid",
+    "ets",
+    "ver",
+    "mid",
+    "actor",
+    "context",
+    "edata"
+  ],
+  "allOf": [
+    {
+      "$ref": "resource:/schemas/telemetry/3.0/common.json"
+    },
+    {
+      "properties": {
+        "eid": {
+          "id": "http://api.ekstep.org/telemetry/eid",
+          "enum": [
+            "SUMMARY"
+          ]
+        },
+        "edata": {
+          "id": "http://api.ekstep.org/telemetry/edata",
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "type",
+            "starttime",
+            "endtime",
+            "timespent",
+            "pageviews",
+            "interactions"
+          ],
+          "properties": {
+            "type": {
+              "id": "http://api.ekstep.org/telemetry/edata/type",
+              "type": "string"
+            },
+            "mode": {
+              "id": "http://api.ekstep.org/telemetry/edata/mode",
+              "type": "string"
+            },
+            "starttime": {
+              "id": "http://api.ekstep.org/telemetry/edata/starttime",
+              "type": "number"
+            },
+            "endtime": {
+              "id": "http://api.ekstep.org/telemetry/edata/endtime",
+              "type": "number"
+            },
+            "timespent": {
+              "id": "http://api.ekstep.org/telemetry/edata/timespent",
+              "type": "number"
+            },
+            "pageviews": {
+              "id": "http://api.ekstep.org/telemetry/edata/pageviews",
+              "type": "number"
+            },
+            "interactions": {
+              "id": "http://api.ekstep.org/telemetry/edata/interactions",
+              "type": "number"
+            },
+            "extra": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "id": {
+                    "id": "http://api.ekstep.org/telemetry/edata/extra/id",
+                    "type": "string"
+                  },
+                  "value": {
+                    "id": "http://api.ekstep.org/telemetry/edata/extra/value",
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala
new file mode 100644
index 0000000000..016f86a087
--- /dev/null
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/PipelinePreprocessorFunction.scala
@@ -0,0 +1,113 @@
+package org.sunbird.dp.preprocessor.functions
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.core.cache.{DedupEngine, RedisConnect}
+import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
+import org.sunbird.dp.preprocessor.domain.Event
+import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig
+import org.sunbird.dp.preprocessor.util.{ShareEventsFlattener, TelemetryValidator}
+
+class PipelinePreprocessorFunction(config: PipelinePreprocessorConfig,
+                                   @transient var telemetryValidator: TelemetryValidator = null,
+                                  @transient var shareEventsFlattener: ShareEventsFlattener = null,
+                                   @transient var dedupEngine: DedupEngine = null)
+                                   (implicit val eventTypeInfo: TypeInformation[Event])
+  extends BaseProcessFunction[Event, Event](config) {
+
+  private[this] val logger = LoggerFactory.getLogger(classOf[PipelinePreprocessorFunction])
+
+  override def metricsList(): List[String] = {
+    List(config.validationFailureMetricsCount,
+      config.validationSuccessMetricsCount,
+      config.duplicationSkippedEventMetricsCount,
+      config.primaryRouterMetricCount,
+      config.logEventsRouterMetricsCount,
+      config.errorEventsRouterMetricsCount,
+      config.auditEventRouterMetricCount,
+      config.shareEventsRouterMetricCount,
+      config.shareItemEventsMetircsCount,
+      config.denormSecondaryEventsRouterMetricsCount,
+      config.denormPrimaryEventsRouterMetricsCount
+    ) ::: deduplicationMetrics
+  }
+
+  override def open(parameters: Configuration): Unit = {
+    super.open(parameters)
+    if (dedupEngine == null) {
+      val redisConnect = new RedisConnect(config.redisHost, config.redisPort, config)
+      dedupEngine = new DedupEngine(redisConnect, config.dedupStore, config.cacheExpirySeconds)
+    }
+
+    if (telemetryValidator == null) {
+      telemetryValidator = new TelemetryValidator(config)
+    }
+
+    if(shareEventsFlattener == null) {
+      shareEventsFlattener = new ShareEventsFlattener(config)
+    }
+  }
+
+  override def close(): Unit = {
+    super.close()
+    dedupEngine.closeConnectionPool()
+  }
+
+  def isDuplicateCheckRequired(producerId: String): Boolean = {
+    config.includedProducersForDedup.contains(producerId)
+  }
+
+  override def processElement(event: Event,
+                              context: ProcessFunction[Event, Event]#Context,
+                              metrics: Metrics): Unit = {
+    val isValid = telemetryValidator.validate(event, context, metrics)
+
+    if (isValid) {
+      if (event.eid().equalsIgnoreCase("LOG")) {
+        context.output(config.logEventsOutputTag, event)
+        metrics.incCounter(metric = config.logEventsRouterMetricsCount)
+      }
+      else {
+        val isUnique = if (isDuplicateCheckRequired(event.producerId())) {
+          deDuplicate[Event, Event](event.mid(), event, context, config.duplicateEventsOutputTag,
+            flagName = config.DEDUP_FLAG_NAME)(dedupEngine, metrics)
+        } else {
+          event.markSkipped(config.DEDUP_SKIP_FLAG_NAME)
+          metrics.incCounter(uniqueEventMetricCount)
+          true
+        }
+
+        if (isUnique) {
+          if("ERROR".equalsIgnoreCase(event.eid())) {
+              metrics.incCounter(metric = config.errorEventsRouterMetricsCount)
+          } else if (config.secondaryEvents.contains(event.eid())) {
+            context.output(config.denormSecondaryEventsRouteOutputTag, event)
+            metrics.incCounter(metric = config.denormSecondaryEventsRouterMetricsCount)
+          }
+          else {
+            context.output(config.denormPrimaryEventsRouteOutputTag, event)
+            metrics.incCounter(metric = config.denormPrimaryEventsRouterMetricsCount)
+          }
+          event.eid() match {
+            case "AUDIT" =>
+              context.output(config.auditRouteEventsOutputTag, event)
+              metrics.incCounter(metric = config.auditEventRouterMetricCount)
+              metrics.incCounter(metric = config.primaryRouterMetricCount) // Since we are are sinking the AUDIT Event into primary router topic
+            case "SHARE" =>
+              shareEventsFlattener.flatten(event, context, metrics)
+              metrics.incCounter(metric = config.shareEventsRouterMetricCount)
+              metrics.incCounter(metric = config.primaryRouterMetricCount) // // Since we are are sinking the SHARE Event into primary router topic
+            case "ERROR" =>
+              context.output(config.errorEventOutputTag, event)
+            case _ => context.output(config.primaryRouteEventsOutputTag, event)
+              metrics.incCounter(metric = config.primaryRouterMetricCount)
+          }
+        }
+      }
+    }
+
+  }
+
+}
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala
index a76ae85627..d7e7b5d6cf 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/functions/ShareEventsFlattenerFunction.scala
@@ -38,12 +38,12 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig)
           val edataType = if (transfers == 0) "download" else "import"
           val shareItemEvent = generateShareItemEvents(shareEvent, EventObject(id = identifier, ver = version,
             `type` = contentType, rollup = Rollup(shareEvent.objectID())), Some(edataType), Some(size))
-          context.output(config.shareItemEventOutputTag, new Gson().toJson(shareItemEvent))
+          context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass)))
         })
       } else {
         val shareItemEvent = generateShareItemEvents(shareEvent, EventObject(id = identifier, ver = version,
           `type` = contentType, rollup = Rollup(shareEvent.objectID())), edataType = Some(shareEvent.edataType()))
-        context.output(config.shareItemEventOutputTag, new Gson().toJson(shareItemEvent))
+        context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass)))
       }
       metrics.incCounter(config.shareItemEventsMetircsCount)
     })
@@ -56,8 +56,8 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig)
                               eventObj: EventObject,
                               edataType: Option[String],
                               paramSize: Option[Double] = None
-                             ): ShareEvent = {
-    ShareEvent(
+                             ): String = {
+    val shareItemEvent = ShareEvent(
       ActorObject(event.actorId(), event.actorType()),
       "SHARE_ITEM",
       EData(event.edataDir, edataType.orNull, paramSize.getOrElse(0)),
@@ -69,6 +69,8 @@ class ShareEventsFlattenerFunction(config: PipelinePreprocessorConfig)
       eventObj,
       event.eventTags
     )
+
+    new Gson().toJson(shareItemEvent)
   }
 
   override def metricsList(): List[String] = {
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala
index 618d53feaf..de6276dc92 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorConfig.scala
@@ -32,6 +32,9 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon
   val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic")
   val kafkaDuplicateTopic: String = config.getString("kafka.output.duplicate.topic")
 
+  val kafkaDenormSecondaryRouteTopic: String = config.getString("kafka.output.denorm.secondary.route.topic")
+  val kafkaDenormPrimaryRouteTopic: String = config.getString("kafka.output.denorm.primary.route.topic")
+
   val defaultChannel: String = config.getString("default.channel")
 
   val includedProducersForDedup: List[String] = config.getStringList("dedup.producer.included.ids").asScala.toList
@@ -44,6 +47,10 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon
   // Router stream out put tags
   val primaryRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("primary-route-events")
 
+  // Spliting events on priority for faster denorm processing
+  val denormSecondaryEventsRouteOutputTag: OutputTag[Event] = OutputTag[Event]("denorm-secondary-events")
+  val denormPrimaryEventsRouteOutputTag: OutputTag[Event] = OutputTag[Event]("denorm-primary-events")
+
   // Audit, Log & Error Events output tag
   val auditRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("audit-route-events")
   val logEventsOutputTag: OutputTag[Event] = OutputTag[Event]("log-route-events")
@@ -51,12 +58,10 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon
 
   // Share events out put tags
   val shareRouteEventsOutputTag: OutputTag[Event] = OutputTag[Event]("share-route-events")
-  val shareItemEventOutputTag: OutputTag[String] = OutputTag[String]("share-item-events")
+  val shareItemEventOutputTag: OutputTag[Event] = OutputTag[Event]("share-item-events")
 
   override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
-  val validationParallelism: Int = config.getInt("task.telemetry.validation.parallelism")
-  val routerParallelism: Int = config.getInt("task.telemetry.router.parallelism")
-  val shareEventsFlattnerParallelism: Int = config.getInt("task.share.events.flattener.parallelism")
+  val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism")
 
   val VALIDATION_FLAG_NAME = "pp_validation_processed"
   val DEDUP_FLAG_NAME = "pp_duplicate"
@@ -69,7 +74,8 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon
   val shareEventsRouterMetricCount = "share-route-success-count"
   val logEventsRouterMetricsCount = "log-route-success-count"
   val errorEventsRouterMetricsCount = "error-route-success-count"
-
+  val denormSecondaryEventsRouterMetricsCount = "denorm-secondary-route-success-count"
+  val denormPrimaryEventsRouterMetricsCount = "denorm-primary-route-success-count"
 
   // Validation job metrics
   val validationSuccessMetricsCount = "validation-success-event-count"
@@ -100,10 +106,12 @@ class PipelinePreprocessorConfig(override val config: Config) extends BaseJobCon
   val auditRouterProducer = "audit-route-sink"
   val invalidEventProducer = "invalid-events-sink"
   val duplicateEventProducer = "duplicate-events-sink"
+  val denormSecondaryEventProducer = "denorm-secondary-events-sink"
+  val denormPrimaryEventProducer = "denorm-primary-events-sink"
 
   val defaultSchemaFile = "envelope.json"
 
-
+  val secondaryEvents: List[String] = config.getStringList("secondary.events").asScala.toList
 
 
 }
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala
index c8006d7153..2f1042d5b5 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/task/PipelinePreprocessorStreamTask.scala
@@ -11,7 +11,7 @@ import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.core.util.FlinkUtil
 import org.sunbird.dp.preprocessor.domain.Event
-import org.sunbird.dp.preprocessor.functions.{ShareEventsFlattenerFunction, TelemetryRouterFunction, TelemetryValidationFunction}
+import org.sunbird.dp.preprocessor.functions.{PipelinePreprocessorFunction, ShareEventsFlattenerFunction, TelemetryRouterFunction, TelemetryValidationFunction}
 
 /**
  * Telemetry Pipeline processor stream task does the following pipeline processing in a sequence:
@@ -61,52 +61,70 @@ class PipelinePreprocessorStreamTask(config: PipelinePreprocessorConfig, kafkaCo
      * 4. Share Events Flattener
      */
 
-    val validationStream: SingleOutputStreamOperator[Event] =
+    val eventStream: SingleOutputStreamOperator[Event] =
       env.addSource(kafkaConsumer, config.pipelinePreprocessorConsumer)
         .uid(config.pipelinePreprocessorConsumer).setParallelism(config.kafkaConsumerParallelism)
         .rebalance()
-        .process(new TelemetryValidationFunction(config)).name(config.telemetryValidationFunction).uid(config.telemetryValidationFunction)
-        .setParallelism(config.validationParallelism)
-
-    val routerStream: SingleOutputStreamOperator[Event] =
-      validationStream.getSideOutput(config.uniqueEventsOutputTag)
-        .process(new TelemetryRouterFunction(config)).name(config.telemetryRouterFunction).uid(config.telemetryRouterFunction)
-        .setParallelism(config.routerParallelism)
-
-    val shareEventsFlattener: SingleOutputStreamOperator[Event] =
-      routerStream.getSideOutput(config.shareRouteEventsOutputTag)
-        .process(new ShareEventsFlattenerFunction(config)).name(config.shareEventsFlattenerFunction).uid(config.shareEventsFlattenerFunction)
-        .setParallelism(config.shareEventsFlattnerParallelism)
+        .process(new PipelinePreprocessorFunction(config)).setParallelism(config.downstreamOperatorsParallelism)
 
     /**
      * Sink for invalid events, duplicate events, log events, audit events and telemetry events
      */
-    validationStream.getSideOutput(config.validationFailedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink(config.kafkaFailedTopic)).name(config.invalidEventProducer).uid(config.invalidEventProducer)
-    validationStream.getSideOutput(config.duplicateEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)).name(config.duplicateEventProducer).uid(config.duplicateEventProducer)
+    eventStream.getSideOutput(config.validationFailedEventsOutputTag).addSink(kafkaConnector.kafkaEventSink(config.kafkaFailedTopic)).name(config.invalidEventProducer).uid(config.invalidEventProducer).setParallelism(config.downstreamOperatorsParallelism)
+    eventStream.getSideOutput(config.duplicateEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDuplicateTopic)).name(config.duplicateEventProducer).uid(config.duplicateEventProducer).setParallelism(config.downstreamOperatorsParallelism)
 
     /**
      * Routing LOG & ERROR Events to "event.log" & "events.error" topic respectively.
      */
-    routerStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaLogRouteTopic)).name(config.logRouterProducer).uid(config.logRouterProducer)
-    routerStream.getSideOutput(config.errorEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaErrorRouteTopic)).name(config.errorRouterProducer).uid(config.errorRouterProducer)
+    eventStream.getSideOutput(config.logEventsOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaLogRouteTopic))
+      .name(config.logRouterProducer).uid(config.logRouterProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
+    eventStream.getSideOutput(config.errorEventOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaErrorRouteTopic))
+      .name(config.errorRouterProducer).uid(config.errorRouterProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     /**
      * Pushing "AUDIT" event into both sink and audit topic
      */
-    routerStream.getSideOutput(config.auditRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaAuditRouteTopic)).name(config.auditRouterProducer).uid(config.auditRouterProducer)
-    routerStream.getSideOutput(config.auditRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.auditEventsPrimaryRouteProducer).uid(config.auditEventsPrimaryRouteProducer)
+    eventStream.getSideOutput(config.auditRouteEventsOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaAuditRouteTopic))
+      .name(config.auditRouterProducer).uid(config.auditRouterProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
+    eventStream.getSideOutput(config.auditRouteEventsOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic))
+      .name(config.auditEventsPrimaryRouteProducer).uid(config.auditEventsPrimaryRouteProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
 
     /**
      * Pushing all the events to unique topic (next stream = denorm) , except LOG, ERROR, AUDIT Events,
      */
-    routerStream.getSideOutput(config.primaryRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.primaryRouterProducer).uid(config.primaryRouterProducer)
+    eventStream.getSideOutput(config.primaryRouteEventsOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic))
+      .name(config.primaryRouterProducer).uid(config.primaryRouterProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
+    /**
+      * Splitting events based on priority and route to different topics (next stream = denorm)
+      */
+    eventStream.getSideOutput(config.denormSecondaryEventsRouteOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDenormSecondaryRouteTopic))
+      .name(config.denormSecondaryEventProducer).uid(config.denormSecondaryEventProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
+    eventStream.getSideOutput(config.denormPrimaryEventsRouteOutputTag)
+      .addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaDenormPrimaryRouteTopic))
+      .name(config.denormPrimaryEventProducer).uid(config.denormPrimaryEventProducer)
+      .setParallelism(config.downstreamOperatorsParallelism)
+
 
     /**
      * Pushing "SHARE and SHARE_ITEM" event into out put topic unique topic(next_streaming_process = denorm)
      */
-
-    shareEventsFlattener.getSideOutput(config.primaryRouteEventsOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.shareEventsPrimaryRouteProducer).uid(config.shareEventsPrimaryRouteProducer)
-    shareEventsFlattener.getSideOutput(config.shareItemEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaPrimaryRouteTopic)).name(config.shareItemsPrimaryRouterProducer).uid(config.shareItemsPrimaryRouterProducer)
+    eventStream.getSideOutput(config.shareItemEventOutputTag).addSink(kafkaConnector.kafkaEventSink[Event](config.kafkaPrimaryRouteTopic)).name(config.shareItemsPrimaryRouterProducer).uid(config.shareItemsPrimaryRouterProducer).setParallelism(config.downstreamOperatorsParallelism)
 
     env.execute(config.jobName)
   }
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala
index c6a6e4a295..550448dfdf 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/SchemaValidator.scala
@@ -1,23 +1,20 @@
 package org.sunbird.dp.preprocessor.util
 
 import java.io.{File, IOException}
-import java.nio.file.{FileSystems, Files, Paths}
+import java.nio.charset.StandardCharsets
+import java.nio.file._
 
 import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import com.github.fge.jackson.JsonLoader
 import com.github.fge.jsonschema.core.exceptions.ProcessingException
 import com.github.fge.jsonschema.core.report.ProcessingReport
 import com.github.fge.jsonschema.main.{JsonSchema, JsonSchemaFactory}
-import com.google.common.io.ByteStreams
-import com.google.gson.Gson
+import io.github.classgraph.{ClassGraph, Resource}
 import org.slf4j.LoggerFactory
 import org.sunbird.dp.preprocessor.domain.Event
 import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig
 
-import scala.collection.JavaConverters._
 import scala.collection.mutable
-import scala.collection.mutable.ListBuffer
-import scala.util.Try
 
 class SchemaValidator(config: PipelinePreprocessorConfig) extends java.io.Serializable {
 
@@ -28,39 +25,29 @@ class SchemaValidator(config: PipelinePreprocessorConfig) extends java.io.Serial
   logger.info("Initializing schema for telemetry objects...")
 
   private val schemaJsonMap: Map[String, JsonSchema] = {
-    val schamaMap = new mutable.HashMap[String, JsonSchema]()
-    val schemaFactory = JsonSchemaFactory.byDefault
-    val schemaUrl = this.getClass.getClassLoader.getResource(s"${config.schemaPath}").toURI
-    // $COVERAGE-OFF$ Disabling code coverage for below code, It can be testable only incase of jar
-    val schemaFiles = if (schemaUrl.getScheme.equalsIgnoreCase("jar")) {
-      val fileSystem = FileSystems.newFileSystem(schemaUrl, Map[String, AnyRef]().asJava)
-      val files = loadSchemaFiles(fileSystem.getPath(s"${config.schemaPath}"))
-      fileSystem.close()
-      files
-    } else {
-      loadSchemaFiles(Paths.get(schemaUrl))
-    }
+    readResourceFiles(s"${config.schemaPath}")
+  }
 
-    logger.info(s"Loaded ${schemaFiles.size} telemetry schema files...")
-    schemaFiles.map { schemaFile =>
-      val schemaJson =
-        new String(ByteStreams.toByteArray(
-          this.getClass.getClassLoader.getResourceAsStream(s"${config.schemaPath}/$schemaFile")
-        ))
-      schamaMap += schemaFile.toString -> schemaFactory.getJsonSchema(JsonLoader.fromString(schemaJson))
+  def readResourceFiles(schemaUrl: String): Map[String, JsonSchema] = {
+    val classGraphResult = new ClassGraph().acceptPaths(schemaUrl).scan()
+    val schemaFactory = JsonSchemaFactory.byDefault
+    val schemaMap = new mutable.HashMap[String, JsonSchema]()
+    try {
+      val resources = classGraphResult.getResourcesWithExtension("json")
+      resources.forEachByteArrayIgnoringIOException((res: Resource, content: Array[Byte]) => {
+        schemaMap += Paths.get(res.getPath).getFileName.toString -> schemaFactory.getJsonSchema(JsonLoader.fromString(new String(content, StandardCharsets.UTF_8)))
+      })
+    } catch {
+      case ex: Exception => ex.printStackTrace()
+        throw ex
+    } finally {
+      classGraphResult.close()
     }
-    schamaMap.toMap
+    schemaMap.toMap
   }
 
   logger.info("Schema initialization completed for telemetry objects...")
 
-  def loadSchemaFiles(schemaDirPath: java.nio.file.Path): List[String] = {
-    val schemaFiles = Try(Files.newDirectoryStream(schemaDirPath)).map { stream =>
-      stream.iterator().asScala.toList.map(path => path.getFileName.toString)
-    }.getOrElse(List[String]())
-    schemaFiles
-  }
-
   def schemaFileExists(event: Event): Boolean = schemaJsonMap.contains(event.schemaName)
 
   @throws[IOException]
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala
new file mode 100644
index 0000000000..085c80ac66
--- /dev/null
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/ShareEventsFlattener.scala
@@ -0,0 +1,91 @@
+package org.sunbird.dp.preprocessor.util
+
+import java.util
+import java.util.UUID
+
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.google.gson.Gson
+import org.sunbird.dp.core.util.JSONUtil
+// import com.google.gson.Gson
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.sunbird.dp.core.job.Metrics
+import org.sunbird.dp.preprocessor.domain._
+import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig
+import org.sunbird.dp.preprocessor.domain.{Context => EventContext}
+
+import scala.util.Try
+
+class ShareEventsFlattener(config: PipelinePreprocessorConfig) extends java.io.Serializable {
+
+  private val serialVersionUID = 1167435095740381669L
+  val objectMapper = new ObjectMapper()
+
+  def flatten(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics) = {
+
+    event.edataItems().forEach(items => {
+      val version = items.get("ver").asInstanceOf[String]
+      val identifier = items.get("id").asInstanceOf[String]
+      val contentType = items.get("type").asInstanceOf[String]
+      val paramsList = Option(items.get("params").asInstanceOf[util.ArrayList[Map[String, AnyRef]]])
+
+      if (paramsList.isDefined) {
+        paramsList.getOrElse(new util.ArrayList[util.Map[String, AnyRef]]()).forEach(
+          param => {
+            val shareEventParams = param.asInstanceOf[util.Map[String, Any]]
+            val transfers = Try(shareEventParams.get("transfers").toString.toDouble).getOrElse(0d)
+            val size = Try(shareEventParams.get("size").toString.toDouble).getOrElse(0d)
+            val edataType = if (transfers == 0) "download" else "import"
+            val shareItemStr = generateShareItemEvents(event, EventObject(id = identifier, ver = version,
+              `type` = contentType, rollup = Rollup(event.objectID())), Some(edataType), Some(size))
+            // context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass)))
+            val shareItemEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](shareItemStr))
+            context.output(config.shareItemEventOutputTag, shareItemEvent)
+            routeShareItemEvents(shareItemEvent, context, metrics)
+          }
+        )
+      } else {
+        val shareItemStr = generateShareItemEvents(event, EventObject(id = identifier, ver = version,
+          `type` = contentType, rollup = Rollup(event.objectID())), edataType = Some(event.edataType()))
+        // context.output(config.shareItemEventOutputTag, new Event(new Gson().fromJson(shareItemEvent, new util.LinkedHashMap[String, Any]().getClass)))
+        val shareItemEvent = new Event(JSONUtil.deserialize[util.LinkedHashMap[String, Any]](shareItemStr))
+        context.output(config.shareItemEventOutputTag, shareItemEvent)
+        routeShareItemEvents(shareItemEvent, context, metrics)
+      }
+      metrics.incCounter(config.shareItemEventsMetircsCount)
+    })
+
+    event.markSuccess(config.SHARE_EVENTS_FLATTEN_FLAG_NAME)
+    context.output(config.primaryRouteEventsOutputTag, event)
+
+  }
+
+
+  def generateShareItemEvents(event: Event, eventObj: EventObject, edataType: Option[String], paramSize: Option[Double] = None): String = {
+    val shareItemEvent = ShareEvent(
+      ActorObject(event.actorId(), event.actorType()),
+      "SHARE_ITEM",
+      EData(event.edataDir, edataType.orNull, paramSize.getOrElse(0)),
+      ver = "3.0",
+      syncts = event.eventSyncTs.asInstanceOf[Number].longValue,
+      ets = event.ets(),
+      context = EventContext(event.channel(), event.env, event.sessionId, event.did(), event.eventPData, event.cdata, event.rollup),
+      mid = event.mid() + "-" + UUID.randomUUID().toString,
+      eventObj,
+      event.eventTags
+    )
+
+    new Gson().toJson(shareItemEvent)
+  }
+
+  def routeShareItemEvents(shareItem: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics) = {
+    if (config.secondaryEvents.contains("SHARE_ITEM")) {
+      context.output(config.denormSecondaryEventsRouteOutputTag, shareItem)
+      metrics.incCounter(metric = config.denormSecondaryEventsRouterMetricsCount)
+    }
+    else {
+      context.output(config.denormPrimaryEventsRouteOutputTag, shareItem)
+      metrics.incCounter(metric = config.denormPrimaryEventsRouterMetricsCount)
+    }
+  }
+
+}
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala
new file mode 100644
index 0000000000..6717a56c51
--- /dev/null
+++ b/data-pipeline-flink/pipeline-preprocessor/src/main/scala/org/sunbird/dp/preprocessor/util/TelemetryValidator.scala
@@ -0,0 +1,64 @@
+package org.sunbird.dp.preprocessor.util
+
+import com.github.fge.jsonschema.core.report.ProcessingReport
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.core.job.Metrics
+import org.sunbird.dp.preprocessor.domain.Event
+import org.sunbird.dp.preprocessor.task.PipelinePreprocessorConfig
+
+class TelemetryValidator(config: PipelinePreprocessorConfig) extends java.io.Serializable {
+
+  private val serialVersionUID = - 2903685244618349843L
+  private[this] val logger = LoggerFactory.getLogger(classOf[TelemetryValidator])
+
+  private val schemaValidator = new SchemaValidator(config)
+
+  def validate(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Boolean = {
+    val isSchemaPresent: Boolean = schemaValidator.schemaFileExists(event)
+    if (isSchemaPresent) {
+      val validationReport = schemaValidator.validate(event, isSchemaPresent = isSchemaPresent)
+      if (validationReport.isSuccess) {
+        onValidationSuccess(event, metrics, context)
+      } else {
+        onValidationFailure(event, metrics, context, validationReport)
+      }
+      validationReport.isSuccess
+    } else {
+      onMissingSchema(event, metrics, context, "Schema not found: eid looks incorrect, sending to failed")
+      false
+    }
+  }
+
+  private def dataCorrection(event: Event): Unit = {
+    // Remove prefix from federated userIds
+    val eventActorId = event.actorId()
+    if (eventActorId != null && !eventActorId.isEmpty && eventActorId.startsWith("f:"))
+      event.updateActorId(eventActorId.substring(eventActorId.lastIndexOf(":") + 1))
+    if (event.objectFieldsPresent && (event.objectType().equalsIgnoreCase("DialCode") || event.objectType().equalsIgnoreCase("qr"))) event.correctDialCodeValue()
+  }
+
+  def onValidationSuccess(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context): Unit = {
+    logger.debug(s"Telemetry schema validation is success: ${event.mid()}")
+    dataCorrection(event)
+    event.markSuccess(config.VALIDATION_FLAG_NAME)
+    metrics.incCounter(config.validationSuccessMetricsCount)
+    event.updateDefaults(config)
+  }
+
+  def onValidationFailure(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context, validationReport: ProcessingReport): Unit = {
+    val failedErrorMsg = schemaValidator.getInvalidFieldName(validationReport.toString)
+    logger.debug(s"Telemetry schema validation is failed for: ${event.mid()} and error message is: ${validationReport.toString}")
+    event.markValidationFailure(failedErrorMsg, config.VALIDATION_FLAG_NAME)
+    metrics.incCounter(config.validationFailureMetricsCount)
+    context.output(config.validationFailedEventsOutputTag, event)
+  }
+
+  def onMissingSchema(event: Event, metrics: Metrics, context: ProcessFunction[Event, Event]#Context, message: String): Unit = {
+    logger.debug(s"Telemetry schema validation is failed for: ${event.mid()} and error message is: $message")
+    event.markValidationFailure(message, config.VALIDATION_FLAG_NAME)
+    metrics.incCounter(config.validationFailureMetricsCount)
+    context.output(config.validationFailedEventsOutputTag, event)
+  }
+
+}
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf b/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf
index 2e1cb89cec..b2d53b3c64 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf
+++ b/data-pipeline-flink/pipeline-preprocessor/src/test/resources/test.conf
@@ -5,21 +5,22 @@ kafka {
   output.failed.topic = "flink.telemetry.failed"
   output.primary.route.topic = "flink.telemetry.unique"
   output.log.route.topic = "flink.druid.events.log"
-  output.error.route.topic = "flink.druid.events.error"
+  output.error.route.topic = "flink.telemetry.error"
   output.audit.route.topic = "flink.telemetry.audit"
   output.duplicate.topic = "flink.telemetry.duplicate"
+  output.denorm.secondary.route.topic = "flink.telemetry.unique.secondary"
+  output.denorm.primary.route.topic = "flink.telemetry.unique.primary"
   groupId = "flink-pipeline-preprocessor-group"
 }
 
 task {
-  telemetry.validation.parallelism = 1
-  telemetry.router.parallelism = 1
-  share.events.flattener.parallelism = 1
+  downstream.operators.parallelism = 1
 }
 
 telemetry.schema.path="schemas/telemetry/3.0"
 default.channel="b00bc992ef25f1a9a8d63291e20efc8d"
 dedup.producer.included.ids = ["dev.sunbird.portal", "dev.sunbird.desktop"]
+secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"]
 
 redis {
   host = 127.0.0.1
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index 7616a76ebf..46f11c6d82 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -123,4 +123,38 @@ object EventFixtures {
       |{"ver":"3.0","ets":1577278681178,"actor":{"type":"User","id":"7c3ea1bb-4da1-48d0-9cc0-c4f150554149"},"context":{"channel":"505c7c48ac6dc1edc9b08f21db5a571d","pdata":{"id":"prod.sunbird.desktop","pid":"sunbird.app","ver":"2.3.162"},"env":"app","sid":"82e41d87-e33f-4269-aeae-d56394985599","did":"1b17c32bad61eb9e33df281eecc727590d739b2b"},"edata":{"dir":"In","type":"File","items":[{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_312785709424099328114191","type":"CONTENT","ver":"1","params":[{"transfers":0,"size":21084308}]},{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_31277435209002188818711","type":"CONTENT","ver":"18","params":[{"transfers":12,"size":"123"}]},{"origin":{"id":"1b17c32bad61eb9e33df281eecc727590d739b2b","type":"Device"},"id":"do_31278794857559654411554","type":"TextBook","ver":"1"}]},"object":{"id":"do_312528116260749312248818","type":"TextBook","version":"10","rollup":{}},"mid":"02ba33e5-15fe-4ec5-b32.1084308E760-3d03429fae84","syncts":1577278682630,"@timestamp":"2019-12-25T12:58:02.630Z","type":"events"}
       |""".stripMargin
 
+  // Invalid SUMMARY event with edata.extra.[].value datatype mismatch
+  val EVENT_13 =
+    """
+      |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":100},{"id":"endpageseen","value":true},{"id":"score","value":2},{"id":"correct","value":2},{"id":"incorrect","value":0},{"id":"partial","value":0},{"id":"skipped","value":0}]}}
+      |
+      |""".stripMargin
+
+  // Invalid SUMMARY event with edata.timespent datatype mismatch
+  val EVENT_14 =
+    """
+      |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":"0:16","pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"},{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"},{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}}
+      |
+      |""".stripMargin
+
+  // Valid SUMMARY event
+  val EVENT_15 =
+    """
+      |{"eid":"SUMMARY","ets":1625043400402,"ver":"3.0","mid":"SUMMARY:a3e517153c4ba392297e70521aa5e17a","actor":{"id":"70e496fe83dee324009a847ea222bc5c","type":"User"},"context":{"channel":"01268904781886259221","pdata":{"id":"staging.sunbird.portal","ver":"4.1.0","pid":"sunbird-portal"},"env":"contentplayer","sid":"73d82044-8ea5-dffc-1af5-6cdf2a1fa1da","did":"70e496fe83dee324009a847ea222bc5c","cdata":[{"id":"kubXMwcsJK2JANa0PeYc00GK5CSXoS1q","type":"ContentSession"},{"id":"xcFG0rntKUGltu2m8zJh7ZqattT9u3Ix","type":"PlaySession"},{"id":"2.0","type":"PlayerVersion"}],"rollup":{"l1":"01268904781886259221"},"uid":"anonymous"},"object":{"id":"do_213302422998196224166","ver":"1","type":"Content","rollup":{}},"tags":["01268904781886259221"],"edata":{"type":"content","mode":"play","starttime":1625043385301,"endtime":1625043401236,"timespent":16,"pageviews":2,"interactions":2,"extra":[{"id":"progress","value":"100"},{"id":"endpageseen","value":"true"},{"id":"score","value":"2"},{"id":"correct","value":"2"},{"id":"incorrect","value":"0"},{"id":"partial","value":"0"},{"id":"skipped","value":"0"}]}}
+      |
+      |""".stripMargin
+
+  // Invalid METRICS event with edata.subsystem missing
+  val EVENT_16 =
+    """
+      |{"@timestamp":"2021-05-09T06:53:58.317Z","actor":{"id":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","type":"User"},"context":{"cdata":[],"did":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","channel":"desktop","env":"DesktopApp","pdata":{"id":"desktop,app","pid":"desktop.app","ver":"1.0"},"sid":"0e2dcdd2-e81c-4e9c-9c43-b41a6fdd6c48"},"edata":{"metrics":[{"metric":"minApiTime","value":0.002482144},{"metric":"maxApiTime","value":7.236265204},{"metric":"avgApiTime","value":0.5468935359999998},{"metric":"totalApiS","value":50.0},{"metric":"minAppStartupTime","value":6.566},{"metric":"maxAppStartupTime","value":11.422},{"metric":"avgAppStartupTime","value":9.755999999999998},{"metric":"totalAppStartupS","value":3.0},{"metric":"minImportTime","value":1.196166354944613},{"metric":"maxImportTime","value":1.196166354944613},{"metric":"avgImportTime","value":1.196166354944613},{"metric":"totalImportS","value":1.0},{"metric":"createdDate","value":1620431999999.0}],"system":"DesktopApp"},"eid":"METRICS","ets":1620542437965.0,"mid":"METRICS:1711d2706507cbd2e22ec88f7660a569","object":{},"syncts":1620543238317,"tags":[],"ver":"3.0"}
+      |
+      |""".stripMargin
+
+  // Valid METRICS event
+  val EVENT_17 =
+    """
+      |{"@timestamp":"2021-05-09T06:53:58.317Z","actor":{"id":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","type":"User"},"context":{"cdata":[],"did":"5e78cb4a62f5c60d3a007ff366c740727afa2aec42f9e63e26fb1a879b87513b","channel":"desktop","env":"DesktopApp","pdata":{"id":"desktop,app","pid":"desktop.app","ver":"1.0"},"sid":"0e2dcdd2-e81c-4e9c-9c43-b41a6fdd6c48"},"edata":{"metrics":[{"metric":"minApiTime","value":0.002482144},{"metric":"maxApiTime","value":7.236265204},{"metric":"avgApiTime","value":0.5468935359999998},{"metric":"totalApiS","value":50.0},{"metric":"minAppStartupTime","value":6.566},{"metric":"maxAppStartupTime","value":11.422},{"metric":"avgAppStartupTime","value":9.755999999999998},{"metric":"totalAppStartupS","value":3.0},{"metric":"minImportTime","value":1.196166354944613},{"metric":"maxImportTime","value":1.196166354944613},{"metric":"avgImportTime","value":1.196166354944613},{"metric":"totalImportS","value":1.0},{"metric":"createdDate","value":1620431999999.0}],"subsystem":"DesktopApp","system":"DesktopApp"},"eid":"METRICS","ets":1620542437965.0,"mid":"METRICS:1711d2706507cbd2e22ec88f7660a569","object":{},"syncts":1620543238317,"tags":[],"ver":"3.0"}
+      |
+      |""".stripMargin
 }
diff --git a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala
index 894b9a5ebd..94ea9dee1b 100644
--- a/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala
+++ b/data-pipeline-flink/pipeline-preprocessor/src/test/scala/org/sunbird/dp/spec/PipelineProcessorStreamTaskTestSpec.scala
@@ -16,10 +16,15 @@ import org.mockito.Mockito.when
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
 import org.sunbird.dp.fixture.EventFixtures
 import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.core.util.JSONUtil
 import org.sunbird.dp.preprocessor.domain.Event
 import org.sunbird.dp.preprocessor.task.{PipelinePreprocessorConfig, PipelinePreprocessorStreamTask}
 import redis.embedded.RedisServer
 
+import scala.collection.JavaConverters._
+
+case class SHARE_ITEM_EVENT(objectId: String, objectType: String)
+
 class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec {
 
   implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
@@ -53,7 +58,9 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec {
     when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaErrorRouteTopic)).thenReturn(new TelemetryErrorEventSink)
     when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaFailedTopic)).thenReturn(new TelemetryFailedEventsSink)
     when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaAuditRouteTopic)).thenReturn(new TelemetryAuditEventSink)
-    when(mockKafkaUtil.kafkaStringSink(ppConfig.kafkaPrimaryRouteTopic)).thenReturn(new ShareItemEventSink)
+
+    when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaDenormSecondaryRouteTopic)).thenReturn(new TelemetryDenormSecondaryEventSink)
+    when(mockKafkaUtil.kafkaEventSink[Event](ppConfig.kafkaDenormPrimaryRouteTopic)).thenReturn(new TelemetryDenormPrimaryEventSink)
 
     flinkCluster.before()
   }
@@ -69,14 +76,18 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec {
     val task = new PipelinePreprocessorStreamTask(ppConfig, mockKafkaUtil)
     task.process()
 
-    ShareItemEventSink.values.size() should be(3)
-    TelemetryPrimaryEventSink.values.size() should be(5)
-    TelemetryFailedEventsSink.values.size() should be(4)
+    // 5 telemetry and 3 SHARE_ITEM
+    TelemetryPrimaryEventSink.values.size() should be(10)
+    TelemetryPrimaryEventSink.values.asScala.count(event => event.eid().equals("SHARE_ITEM")) should be (3)
+    TelemetryFailedEventsSink.values.size() should be(7)
     DupEventsSink.values.size() should be(1)
     TelemetryAuditEventSink.values.size() should be(1)
     TelemetryLogEventSink.values.size() should be(1)
     TelemetryErrorEventSink.values.size() should be(1)
 
+    TelemetryDenormSecondaryEventSink.values.size() should be(4) // 1 INTERACT and 3 SHARE_ITEM
+    TelemetryDenormPrimaryEventSink.values.size() should be(6)
+
     /**
      * * 1. primary-route-success-count -> 05
      * * 2. audit-route-success-count -> 01
@@ -90,22 +101,38 @@ class PipelineProcessorStreamTaskTestSpec extends BaseTestSpec {
      * * 10. unique-event-count -> 02
      * * 12. share-item-event-success-count -> 03
      */
-    
-    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.primaryRouterMetricCount}").getValue() should be(5)
+    val expectedShareItems: List[SHARE_ITEM_EVENT] = List(
+      SHARE_ITEM_EVENT(objectId = "do_312785709424099328114191", objectType = "CONTENT"),
+      SHARE_ITEM_EVENT(objectId = "do_31277435209002188818711", objectType = "CONTENT"),
+      SHARE_ITEM_EVENT(objectId = "do_31278794857559654411554", objectType = "TextBook")
+    )
+
+    val shareItems = TelemetryPrimaryEventSink.values.asScala.filter(event => event.eid().equals("SHARE_ITEM"))
+    shareItems.foreach {
+      event =>
+        val shareItemObject = event.getTelemetry.read[util.HashMap[String, AnyRef]]("object").getOrElse(new util.HashMap()).asScala
+        val actualShareItem = SHARE_ITEM_EVENT(objectId = shareItemObject("id").asInstanceOf[String], shareItemObject("type").asInstanceOf[String])
+        expectedShareItems should contain (actualShareItem)
+    }
+
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.primaryRouterMetricCount}").getValue() should be(7)
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.shareItemEventsMetircsCount}").getValue() should be(3)
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.auditEventRouterMetricCount}").getValue() should be(1)
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.shareEventsRouterMetricCount}").getValue() should be(1)
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.logEventsRouterMetricsCount}").getValue() should be(1)
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.errorEventsRouterMetricsCount}").getValue() should be(1)
 
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationSuccessMetricsCount}").getValue() should be(10)
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationFailureMetricsCount}").getValue() should be(7)
 
-    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationSuccessMetricsCount}").getValue() should be(8)
-    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.validationFailureMetricsCount}").getValue() should be(4)
-
-    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.unique-event-count").getValue() should be(7)
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.unique-event-count").getValue() should be(8) // ONLY LOG events are skipped from dedup
     BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.duplicate-event-count").getValue() should be(1)
+
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.denormSecondaryEventsRouterMetricsCount}").getValue() should be(4)
+    BaseMetricsReporter.gaugeMetrics(s"${ppConfig.jobName}.${ppConfig.denormPrimaryEventsRouterMetricsCount}").getValue() should be(6)
+
+  }
   }
-}
 
 class PipeLineProcessorEventSource extends SourceFunction[Event] {
 
@@ -123,6 +150,11 @@ class PipeLineProcessorEventSource extends SourceFunction[Event] {
     val event10 = gson.fromJson(EventFixtures.EVENT_10, new util.LinkedHashMap[String, Any]().getClass)
     val event11 = gson.fromJson(EventFixtures.EVENT_11, new util.LinkedHashMap[String, Any]().getClass)
     val event12 = gson.fromJson(EventFixtures.EVENT_12, new util.LinkedHashMap[String, Any]().getClass)
+    val event13 = gson.fromJson(EventFixtures.EVENT_13, new util.LinkedHashMap[String, Any]().getClass)
+    val event14 = gson.fromJson(EventFixtures.EVENT_14, new util.LinkedHashMap[String, Any]().getClass)
+    val event15 = gson.fromJson(EventFixtures.EVENT_15, new util.LinkedHashMap[String, Any]().getClass)
+    val event16 = gson.fromJson(EventFixtures.EVENT_16, new util.LinkedHashMap[String, Any]().getClass)
+    val event17 = gson.fromJson(EventFixtures.EVENT_17, new util.LinkedHashMap[String, Any]().getClass)
     ctx.collect(new Event(event1))
     ctx.collect(new Event(event2))
     ctx.collect(new Event(event3))
@@ -135,25 +167,17 @@ class PipeLineProcessorEventSource extends SourceFunction[Event] {
     ctx.collect(new Event(event10))
     ctx.collect(new Event(event11))
     ctx.collect(new Event(event12))
+    ctx.collect(new Event(event13))
+    ctx.collect(new Event(event14))
+    ctx.collect(new Event(event15))
+    ctx.collect(new Event(event16))
+    ctx.collect(new Event(event17))
   }
 
   override def cancel() = {}
 
 }
 
-class ShareItemEventSink extends SinkFunction[String] {
-
-  override def invoke(value: String): Unit = {
-    synchronized {
-      ShareItemEventSink.values.add(value)
-    }
-  }
-}
-
-object ShareItemEventSink {
-  val values: util.List[String] = new util.ArrayList()
-}
-
 class TelemetryFailedEventsSink extends SinkFunction[Event] {
 
   override def invoke(value: Event): Unit = {
@@ -233,4 +257,30 @@ class DupEventsSink extends SinkFunction[Event] {
 
 object DupEventsSink {
   val values: util.List[Event] = new util.ArrayList()
+}
+
+class TelemetryDenormSecondaryEventSink extends SinkFunction[Event] {
+
+  override def invoke(value: Event): Unit = {
+    synchronized {
+      TelemetryDenormSecondaryEventSink.values.add(value)
+    }
+  }
+}
+
+object TelemetryDenormSecondaryEventSink {
+  val values: util.List[Event] = new util.ArrayList()
+}
+
+class TelemetryDenormPrimaryEventSink extends SinkFunction[Event] {
+
+  override def invoke(value: Event): Unit = {
+    synchronized {
+      TelemetryDenormPrimaryEventSink.values.add(value)
+    }
+  }
+}
+
+object TelemetryDenormPrimaryEventSink {
+  val values: util.List[Event] = new util.ArrayList()
 }
\ No newline at end of file
diff --git a/data-pipeline-flink/pom.xml b/data-pipeline-flink/pom.xml
index 9bd5dd9139..51ac28a286 100644
--- a/data-pipeline-flink/pom.xml
+++ b/data-pipeline-flink/pom.xml
@@ -21,26 +21,28 @@
 
 	<modules>
 		<module>dp-core</module>
+		<module>ingest-router</module>
 		<module>telemetry-extractor</module>
 		<module>pipeline-preprocessor</module>
 		<module>de-normalization</module>
 		<module>druid-events-validator</module>
 		<module>device-profile-updater</module>
-		<module>user-cache-updater</module>
 		<module>content-cache-updater</module>
 		<module>assessment-aggregator</module>
+		<module>user-cache-updater-2.0</module>
 	</modules>
 
 	<properties>
 		<!-- maven specific properties -->
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<scala.version>2.12</scala.version>
-		<scala.maj.version>2.12.11</scala.maj.version>
-		<flink.version>1.10.1</flink.version>
+		<scala.maj.version>2.12</scala.maj.version>
+		<scala.version>2.12.11</scala.version>
+		<flink.version>1.13.5</flink.version>
 		<kafka.version>2.4.0</kafka.version>
+		<java.target.runtime>11</java.target.runtime>
 		<jackson-jaxrs.version>1.9.13</jackson-jaxrs.version>
 		<scoverage.plugin.version>1.4.0</scoverage.plugin.version>
-		<release-version>release-3.0.2</release-version>
+		<release-version>release-4.6.0</release-version>
 	</properties>
 
 	<repositories>
@@ -73,10 +75,10 @@
 				<plugin>
 					<groupId>org.apache.maven.plugins</groupId>
 					<artifactId>maven-compiler-plugin</artifactId>
-					<version>3.1</version>
+					<version>3.8.1</version>
 					<configuration>
-						<source>1.8</source>
-						<target>1.8</target>
+						<source>${java.target.runtime}</source>
+						<target>${java.target.runtime}</target>
 					</configuration>
 				</plugin>
 				<plugin>
diff --git a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile
index b00c143bbe..c220a7daa1 100644
--- a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile
+++ b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile
@@ -1,7 +1,9 @@
-FROM anandp504/flink:1.10.1-scala_2.12
+FROM sunbird/flink:1.13.5-scala_2.12-java11
 COPY target/sunbird-dp-distribution-1.0.tar.gz /tmp
 USER flink
 RUN tar -xvf /tmp/sunbird-dp-distribution-1.0.tar.gz -C $FLINK_HOME/lib/
+RUN mkdir $FLINK_HOME/plugins/s3-fs-presto
+RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/plugins/s3-fs-presto/
 USER root
 RUN rm -f /tmp/sunbird-dp-distribution-1.0.tar.gz
 USER flink
diff --git a/data-pipeline-flink/sunbird-dp-distribution/pom.xml b/data-pipeline-flink/sunbird-dp-distribution/pom.xml
index 2cc9a4c876..1852694288 100644
--- a/data-pipeline-flink/sunbird-dp-distribution/pom.xml
+++ b/data-pipeline-flink/sunbird-dp-distribution/pom.xml
@@ -16,10 +16,16 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <type>jar</type>
         </dependency>
+        <dependency>
+            <groupId>org.sunbird.dp.jobs</groupId>
+            <artifactId>ingest-router</artifactId>
+            <version>1.0.0</version>
+            <type>jar</type>
+        </dependency>
         <dependency>
             <groupId>org.sunbird.dp.jobs</groupId>
             <artifactId>telemetry-extractor</artifactId>
@@ -52,8 +58,8 @@
         </dependency>
         <dependency>
             <groupId>org.sunbird.dp.jobs</groupId>
-            <artifactId>user-cache-updater</artifactId>
-            <version>1.0.0</version>
+            <artifactId>user-cache-updater-2.0</artifactId>
+            <version>2.0.0</version>
             <type>jar</type>
         </dependency>
         <dependency>
diff --git a/data-pipeline-flink/telemetry-extractor/pom.xml b/data-pipeline-flink/telemetry-extractor/pom.xml
index 6776dfbfed..5c9be58489 100644
--- a/data-pipeline-flink/telemetry-extractor/pom.xml
+++ b/data-pipeline-flink/telemetry-extractor/pom.xml
@@ -27,7 +27,7 @@
     <dependencies>
         <dependency>
             <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
             <version>${flink.version}</version>
             <scope>provided</scope>
         </dependency>
@@ -46,13 +46,13 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -65,7 +65,7 @@
         <dependency>
             <groupId>org.apache.flink</groupId>
             <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
+            <version>${flink.version}</version>
             <scope>test</scope>
             <classifier>tests</classifier>
         </dependency>
@@ -93,6 +93,14 @@
     		<sourceDirectory>src/main/scala</sourceDirectory>
 		    <testSourceDirectory>src/test/scala</testSourceDirectory>
         <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
@@ -141,10 +149,11 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
+                <version>4.4.0</version>
                 <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
                     <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
                 </configuration>
                 <executions>
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties b/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties
deleted file mode 100644
index dda4b21056..0000000000
--- a/data-pipeline-flink/telemetry-extractor/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=telemetry-extractor.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf b/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf
index f4801f93b7..9518da6df3 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf
+++ b/data-pipeline-flink/telemetry-extractor/src/main/resources/telemetry-extractor.conf
@@ -3,6 +3,7 @@ include "base-config.conf"
 kafka {
   input.topic = ${job.env}".telemetry.ingest"
   output.success.topic = ${job.env}".telemetry.raw"
+  output.log.route.topic = ${job.env}".druid.events.log"
   output.duplicate.topic = ${job.env}".telemetry.extractor.duplicate"
   output.failed.topic = ${job.env}".telemetry.failed"
   output.batch.failed.topic = ${job.env}".telemetry.extractor.failed"
@@ -16,9 +17,7 @@ kafka {
 
 task {
   consumer.parallelism = 1
-  dedup.parallelism = 1
-  extraction.parallelism = 1
-  redactor.parallelism = 1
+  downstream.operators.parallelism = 1
 }
 
 redis {
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala
index 471f04a9c3..fa4dbda67e 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/domain/Models.scala
@@ -12,14 +12,5 @@ case class Pdata(ver: String, pid: String, id: String = "data-pipeline")
 
 case class  Object(id:String, ver: String, `type`:String, rollup: Option[Map[String, String]])
 
-case class LogEvent(actor: Actor,
-                    eid: String,
-                    edata: EData,
-                    ver: String = "3.0",
-                    syncts: Long,
-                    ets: Long = System.currentTimeMillis(),
-                    context: Context,
-                    mid: String,
-                    `object`: Object,
-                    tags: Seq[AnyRef]
-                   )
\ No newline at end of file
+case class LogEvent(actor: Actor, eid: String, edata: EData, ver: String = "3.0", syncts: Long, ets: Long = System.currentTimeMillis(),
+                    context: Context, mid: String, `object`: Object, tags: Seq[AnyRef])
\ No newline at end of file
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala
index 8e305286d6..d9d5d242d9 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/DeduplicationFunction.scala
@@ -2,7 +2,8 @@ package org.sunbird.dp.extractor.functions
 
 import java.util
 
-import com.google.gson.Gson
+import org.sunbird.dp.core.util.JSONUtil
+// import com.google.gson.Gson
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.configuration.Configuration
 import org.apache.flink.streaming.api.functions.ProcessFunction
@@ -53,13 +54,15 @@ class DeduplicationFunction(config: TelemetryExtractorConfig, @transient var ded
         throw jedisEx
       }
       case ex: Exception => {
+        logger.error("Unexpected Error", ex)
         metrics.incCounter(config.failedBatchCount)
         context.output(config.failedBatchEventOutputTag, batchEvents)
       }
     }
 
     def getMsgIdentifier(batchEvents: String): String = {
-      val event = new Gson().fromJson(batchEvents, new util.LinkedHashMap[String, AnyRef]().getClass)
+      // val event = new Gson().fromJson(batchEvents, new util.LinkedHashMap[String, AnyRef]().getClass)
+      val event = JSONUtil.deserialize[util.LinkedHashMap[String, AnyRef]](batchEvents)
       val paramsObj = Option(event.get("params"))
       val messageId = paramsObj.map {
         params => params.asInstanceOf[util.Map[String, AnyRef]].get("msgid").asInstanceOf[String]
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala
index e9091b1722..97b7ddd8a1 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/functions/ExtractionFunction.scala
@@ -11,6 +11,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.streaming.api.functions.ProcessFunction
 import org.joda.time.format.DateTimeFormat
 import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
+import org.sunbird.dp.core.util.JSONUtil
 import org.sunbird.dp.extractor.domain._
 import org.sunbird.dp.extractor.domain.{Context => EventContext}
 import org.sunbird.dp.extractor.task.TelemetryExtractorConfig
@@ -40,25 +41,28 @@ class ExtractionFunction(config: TelemetryExtractorConfig)(implicit val stringTy
     eventsList.forEach(event => {
       val eventId = event.get("eid").asInstanceOf[String]
       val eventData = updateEvent(event, syncTs)
-      val eventJson = gson.toJson(eventData)
+      val eventJson = JSONUtil.serialize(eventData)
       val eventSize = eventJson.getBytes("UTF-8").length
       if (eventSize > config.eventMaxSize) {
         metrics.incCounter(config.failedEventCount)
         context.output(config.failedEventsOutputTag, markFailed(eventData))
       } else {
         metrics.incCounter(config.successEventCount)
-        if (config.redactEventsList.contains(eventId))
+        if (config.redactEventsList.contains(eventId)) {
           context.output(config.assessRedactEventsOutputTag, markSuccess(eventData))
-        else
+        } else if ("LOG".equalsIgnoreCase(eventId)) {
+          context.output(config.logEventsOutputTag, markSuccess(eventData))
+        } else {
           context.output(config.rawEventsOutputTag, markSuccess(eventData))
+        }
       }
     })
 
     /**
      * Generating Audit events to compute the number of events in the batch.
      */
-    context.output(config.logEventsOutputTag,
-      gson.fromJson(gson.toJson(generateAuditEvents(eventsList.size(), batchEvent)), mapType))
+    // context.output(config.logEventsOutputTag, gson.fromJson(gson.toJson(generateAuditEvents(eventsList.size(), batchEvent)), mapType))
+    context.output(config.auditEventsOutputTag, gson.toJson(generateAuditEvents(eventsList.size(), batchEvent)))
     metrics.incCounter(config.auditEventCount)
 
   }
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala
index a3f6003982..6ff583997c 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorConfig.scala
@@ -22,6 +22,7 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi
   // Kafka Topics Configuration
   val kafkaInputTopic: String = config.getString("kafka.input.topic")
   val kafkaSuccessTopic: String = config.getString("kafka.output.success.topic")
+  val kafkaLogRouteTopic: String = config.getString("kafka.output.log.route.topic")
   val kafkaDuplicateTopic: String = config.getString("kafka.output.duplicate.topic")
   val kafkaFailedTopic: String = config.getString("kafka.output.failed.topic")
   val kafkaBatchFailedTopic: String = config.getString("kafka.output.batch.failed.topic")
@@ -29,21 +30,21 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi
   val eventMaxSize: Long = config.getLong("kafka.event.max.size")
 
   override val kafkaConsumerParallelism: Int = config.getInt("task.consumer.parallelism")
-  val deDupParallelism: Int = config.getInt("task.dedup.parallelism")
-  val extractionParallelism: Int = config.getInt("task.extraction.parallelism")
-  val redactorParallelism: Int = config.getInt("task.redactor.parallelism")
+  val downstreamOperatorsParallelism: Int = config.getInt("task.downstream.operators.parallelism")
 
   val redactEventsList: List[String] = config.getStringList("redact.events.list").asScala.toList
   val contentStore: Int = config.getInt("redis-meta.database.contentstore.id")
 
-  val UNIQUE_EVENTS_OUTPUT_TAG = "unique-events"
+  val UNIQUE_EVENTS_OUTPUT_TAG = "unique-batch-events"
   val RAW_EVENTS_OUTPUT_TAG = "raw-events"
+  val LOG_EVENTS_OUTPUT_TAG = "log-events"
+  val ERROR_EVENTS_OUTPUT_TAG = "error-events"
   val ASSESS_REDACT_EVENTS_OUTPUT_TAG = "assess-redact-events"
   val ASSESS_RAW_EVENTS_OUTPUT_TAG = "assess-raw-events"
   val FAILED_EVENTS_OUTPUT_TAG = "failed-events"
   val FAILED_BATCH_EVENTS_OUTPUT_TAG = "failed-batch-events"
-  val LOG_EVENTS_OUTPUT_TAG = "log-events"
-  val DUPLICATE_EVENTS_OUTPUT_TAG = "duplicate-events"
+  val AUDIT_EVENTS_OUTPUT_TAG = "audit-events"
+  val DUPLICATE_EVENTS_OUTPUT_TAG = "duplicate-batch-events"
 
   // Metric List
   val successEventCount = "success-event-count"
@@ -56,11 +57,13 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi
   val skippedEventCount = "skipped-event-count"
 
   val rawEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](RAW_EVENTS_OUTPUT_TAG)
+  val logEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](LOG_EVENTS_OUTPUT_TAG)
+  val errorEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ERROR_EVENTS_OUTPUT_TAG)
   val assessRawEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ASSESS_RAW_EVENTS_OUTPUT_TAG)
   val assessRedactEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](ASSESS_REDACT_EVENTS_OUTPUT_TAG)
   val failedEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](FAILED_EVENTS_OUTPUT_TAG)
   val failedBatchEventOutputTag: OutputTag[String] = OutputTag[String](FAILED_BATCH_EVENTS_OUTPUT_TAG)
-  val logEventsOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](LOG_EVENTS_OUTPUT_TAG)
+  val auditEventsOutputTag: OutputTag[String] = OutputTag[String](AUDIT_EVENTS_OUTPUT_TAG)
   val duplicateEventOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](id = DUPLICATE_EVENTS_OUTPUT_TAG)
   val uniqueEventOutputTag: OutputTag[util.Map[String, AnyRef]] = OutputTag[util.Map[String, AnyRef]](id = UNIQUE_EVENTS_OUTPUT_TAG)
 
@@ -76,6 +79,8 @@ class TelemetryExtractorConfig(override val config: Config) extends BaseJobConfi
   val extractorBatchFailedEventsProducer = "extractor-batch-failed-events-sink"
   val extractorRawEventsProducer = "extractor-raw-events-sink"
   val extractorAuditEventsProducer = "extractor-audit-events-sink"
+  val extractorLogEventsProducer = "extractor-log-events-sink"
+  val extractorErrorEventsProducer = "extractor-error-events-sink"
   val extractorFailedEventsProducer = "extractor-failed-events-sink"
   val assessEventsProducer = "assess-events-sink"
   val assessRawEventsProducer = "assess-raw-events-sink"
diff --git a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala
index c4a9ebed62..e0e226f843 100644
--- a/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/main/scala/org/sunbird/dp/extractor/task/TelemetryExtractorStreamTask.scala
@@ -61,7 +61,7 @@ class TelemetryExtractorStreamTask(config: TelemetryExtractorConfig, kafkaConnec
         .rebalance()
         .process(new DeduplicationFunction(config))
         .name("ExtractorDeduplicationFn").uid("ExtractorDeduplicationFn")
-        .setParallelism(config.deDupParallelism)
+        .setParallelism(config.downstreamOperatorsParallelism)
     /**
      * After - De-Duplication process.
      *  1. Extract the batch events.
@@ -71,22 +71,36 @@ class TelemetryExtractorStreamTask(config: TelemetryExtractorConfig, kafkaConnec
       deDupStream.getSideOutput(config.uniqueEventOutputTag)
         .process(new ExtractionFunction(config))
         .name(config.extractionFunction).uid(config.extractionFunction)
-        .setParallelism(config.extractionParallelism)
+        .setParallelism(config.downstreamOperatorsParallelism)
 
     val redactorStream =
       extractionStream.getSideOutput(config.assessRedactEventsOutputTag)
         .process(new RedactorFunction(config)).name(config.redactorFunction).uid(config.redactorFunction)
-        .setParallelism(config.redactorParallelism)
+        .setParallelism(config.downstreamOperatorsParallelism)
 
-    deDupStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaDuplicateTopic)).name(config.extractorDuplicateProducer).uid(config.extractorDuplicateProducer)
-    deDupStream.getSideOutput(config.failedBatchEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaBatchFailedTopic)).name(config.extractorBatchFailedEventsProducer).uid(config.extractorBatchFailedEventsProducer)
+    deDupStream.getSideOutput(config.duplicateEventOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaDuplicateTopic))
+      .name(config.extractorDuplicateProducer).uid(config.extractorDuplicateProducer).setParallelism(config.downstreamOperatorsParallelism)
 
-    extractionStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.extractorRawEventsProducer).uid(config.extractorRawEventsProducer)
-    extractionStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.extractorAuditEventsProducer).uid(config.extractorAuditEventsProducer)
-    extractionStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaFailedTopic)).name(config.extractorFailedEventsProducer).uid(config.extractorFailedEventsProducer)
+    deDupStream.getSideOutput(config.failedBatchEventOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaBatchFailedTopic))
+      .name(config.extractorBatchFailedEventsProducer).uid(config.extractorBatchFailedEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
 
-    redactorStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic)).name(config.assessEventsProducer).uid(config.assessEventsProducer)
-    redactorStream.getSideOutput(config.assessRawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaAssessRawTopic)).name(config.assessRawEventsProducer).uid(config.assessRawEventsProducer)
+    extractionStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic))
+      .name(config.extractorRawEventsProducer).uid(config.extractorRawEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
+
+    extractionStream.getSideOutput(config.logEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaLogRouteTopic))
+      .name(config.extractorLogEventsProducer).uid(config.extractorLogEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
+
+    extractionStream.getSideOutput(config.auditEventsOutputTag).addSink(kafkaConnector.kafkaStringSink(config.kafkaLogRouteTopic))
+      .name(config.extractorAuditEventsProducer).uid(config.extractorAuditEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
+
+    extractionStream.getSideOutput(config.failedEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaFailedTopic))
+      .name(config.extractorFailedEventsProducer).uid(config.extractorFailedEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
+
+    redactorStream.getSideOutput(config.rawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaSuccessTopic))
+      .name(config.assessEventsProducer).uid(config.assessEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
+
+    redactorStream.getSideOutput(config.assessRawEventsOutputTag).addSink(kafkaConnector.kafkaMapSink(config.kafkaAssessRawTopic))
+      .name(config.assessRawEventsProducer).uid(config.assessRawEventsProducer).setParallelism(config.downstreamOperatorsParallelism)
 
     env.execute(config.jobName)
   }
diff --git a/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf b/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf
index 1c37949cb0..cb85dc1f76 100644
--- a/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf
+++ b/data-pipeline-flink/telemetry-extractor/src/test/resources/test.conf
@@ -3,6 +3,7 @@ include "base-test.conf"
 kafka {
   input.topic = "flink.telemetry.ingest"
   output.success.topic = "flink.telemetry.raw"
+  output.log.route.topic = "flink.druid.events.log"
   output.duplicate.topic = "flink.telemetry.extractor.duplicate"
   output.failed.topic = "flink.telemetry.failed"
   output.batch.failed.topic = "flink.telemetry.extractor.failed"
@@ -13,9 +14,7 @@ kafka {
 
 task {
   consumer.parallelism = 1
-  dedup.parallelism = 1
-  extraction.parallelism = 1
-  redactor.parallelism = 1
+  downstream.operators.parallelism = 1
 }
 
 redis {
diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala
new file mode 100644
index 0000000000..64caa9cfe1
--- /dev/null
+++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/Event.scala
@@ -0,0 +1,9 @@
+package org.sunbird.dp.fixture
+
+import java.util
+
+import org.sunbird.dp.core.domain.Events
+
+class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
+
+}
diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
index 774c80756e..894859bf67 100644
--- a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -4,12 +4,12 @@ object EventFixture {
 
   val EVENT_WITH_MESSAGE_ID: String =
     """
-      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"ERROR","ets":1604733538282,"ver":"3.0","mid":"ERROR:36855629","actor":{"id":"479ea1e9","type":"User"},"context":{"channel":"1234","pdata":{"id":"local.diksha.app","ver":"3.2.485","pid":"local.app.contentplayer"},"env":"contentplayer","sid":"f55869e7-1f87","did":"a3aa6eed374c1a","cdata":[{"id":"streaming","type":"PlayerLaunch"},{"id":"a29d203eb","type":"ContentSession"},{"id":"bbc96db","type":"PlaySession"},{"id":"f55869e7-1","type":"UserSession"}],"rollup":{"l1":"0126684405014522"}},"object":{"id":"do_313134032380190720","type":"Content","ver":"1","rollup":{"l1":"do_3131397259090575361"}},"tags":[],"edata":{"err":"Missing PDF","errtype":"CONTENT","stacktrace":"Error\n    at MissingPDFExceptionClosure","pageid":""}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
       |""".stripMargin
 
   val EVENT_WITHOUT_MESSAGE_ID: String =
     """
-      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
+      |{"id":"sunbird.telemetry","ver":"3.0","ets":1529500243591,"params":{"requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{"id":"do_21299582901864857613016","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test1","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_21299582901864857613016","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test2","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"do_312526125187809280139355","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"ASSESS","ets":1.586431492513E12,"ver":"3.1","mid":"ASSESS:12159f2827880221eef12a6be9560379:test3","actor":{"id":"6e89dba6-10d6-4044-9105-b80ce7f56b38","type":"User"},"context":{"channel":"01275678925675724817","pdata":{"id":"preprod.diksha.app","ver":"2.8.260preproduction","pid":"sunbird.app.contentplayer"},"env":"contentplayer","sid":"0cbc018c","did":"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8","cdata":[]},"object":{"id":"do_212995828601487360194","type":"Content","ver":"2"},"tags":[],"edata":{"item":{"id":"test-item-id","maxscore":1.0,"type":"ftb","exlength":0.0,"params":[{"eval":"order"}],"uri":"","title":"Registration","mmc":[],"mc":[],"desc":""},"index":1.0,"pass":"Yes","score":1.0,"resvalues":[{"1":"{\"text\":\"NARENDRA MODI\"}"}],"duration":2.0},"syncts":1.586431504608E12,"@timestamp":"2020-04-09T11:25:04.608Z"},{"eid":"RESPONSE","ets":1.586371994743E12,"ver":"3.0","mid":"RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc","actor":{"id":"5ffb95d1-5928-4e03-82dd-d9e527c7b650","type":"User"},"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"DjZovFRz9DRGDCU6NzyqsZ51cfJY2Ha1","did":"701bc22009b6d4fdf9122ad4d8f966f2","cdata":[{"id":"do_21299554280196505612827","type":"course"}],"rollup":{"l1":"0126796199493140480"}},"object":{"id":"do_2129493337594429441162","type":"Content","ver":"1"},"tags":["0126796199493140480"],"edata":{"target":{},"type":"CHOOSE","values":[{"option1":"\u003cp\u003eSabha\u003c/p\u003e\n"}]},"syncts":1586371998452,"@timestamp":"2020-04-08T18:53:18.452Z"}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
       |""".stripMargin
 
   val EMPTY_BATCH_EVENTS: String =
diff --git a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala
index 2a67f7897d..a0a8afe6e9 100644
--- a/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala
+++ b/data-pipeline-flink/telemetry-extractor/src/test/scala/org/sunbird/dp/spec/TelemetryExtractionStreamTaskTestSpec.scala
@@ -8,7 +8,7 @@ import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.java.typeutils.TypeExtractor
 import org.mockito.Mockito
 import org.mockito.Mockito._
-import org.sunbird.dp.fixture.EventFixture
+import org.sunbird.dp.fixture.{Event, EventFixture}
 import redis.embedded.RedisServer
 import org.apache.flink.streaming.api.functions.sink.SinkFunction
 import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
@@ -17,6 +17,7 @@ import org.apache.flink.streaming.api.functions.source.SourceFunction
 import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
 import com.typesafe.config.ConfigFactory
 import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.core.domain.Events
 import org.sunbird.dp.core.job.FlinkKafkaConnector
 import org.sunbird.dp.extractor.task.{TelemetryExtractorConfig, TelemetryExtractorStreamTask}
 import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
@@ -48,11 +49,13 @@ class ExtractionStreamTaskTestSpec extends BaseTestSpec {
     when(mockKafkaUtil.kafkaStringSource(extractorConfig.kafkaInputTopic)).thenReturn(new ExtractorEventSource)
     when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaDuplicateTopic)).thenReturn(new DupEventsSink)
     when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaSuccessTopic)).thenReturn(new RawEventsSink)
+    when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaLogRouteTopic)).thenReturn(new LogEventsSink)
+    when(mockKafkaUtil.kafkaStringSink(extractorConfig.kafkaLogRouteTopic)).thenReturn(new AuditEventsSink)
     when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaFailedTopic)).thenReturn(new FailedEventsSink)
     when(mockKafkaUtil.kafkaStringSink(extractorConfig.kafkaBatchFailedTopic)).thenReturn(new FailedBatchEventsSink)
     when(mockKafkaUtil.kafkaMapSink(extractorConfig.kafkaAssessRawTopic)).thenReturn(new AssessRawEventsSink)
 
-    setupRedisTestData
+    setupRedisTestData()
     flinkCluster.before()
   }
 
@@ -78,31 +81,50 @@ class ExtractionStreamTaskTestSpec extends BaseTestSpec {
     val task = new TelemetryExtractorStreamTask(extractorConfig, mockKafkaUtil)
     task.process()
 
-    RawEventsSink.values.size() should be (45) // 43 events + 2 log events generated for auditing
+    // RawEventsSink.values.size() should be (43) // 43 events + 2 log events generated for auditing
+    RawEventsSink.values.size() should be (6) // includes 1 Error Event
+    AuditEventsSink.values.size() should be (2)
+    LogEventsSink.values.size() should be (38)
     FailedEventsSink.values.size() should be (2)
     FailedBatchEventsSink.values.size() should be (1)
     DupEventsSink.values.size() should be (1)
     AssessRawEventsSink.values.size() should be (2)
 
-    val rawEvent = gson.fromJson(gson.toJson(RawEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
+    // val rawEvent = gson.fromJson(gson.toJson(RawEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
+    val rawEvent = RawEventsSink.values.get(0)
     val dupEvent = gson.fromJson(gson.toJson(DupEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
     val failedEvent = gson.fromJson(gson.toJson(FailedEventsSink.values.get(0)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
-    rawEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true)
+    // rawEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true)
+    rawEvent.flags().asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(true)
     dupEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("extractor_duplicate") should be(true)
     failedEvent("flags").asInstanceOf[util.Map[String, Boolean]].get("ex_processed") should be(false)
 
     // Assertions for redactor logic
+    val responseEventWithoutValues = RawEventsSink.values.asScala.toList.filter(ev => ev.mid().equals("RESPONSE:4dd15933b5b8230deda7c4f67d8b61fc")).head
+    responseEventWithoutValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("values").asInstanceOf[util.ArrayList[AnyRef]].size should be (0)
+
+    val assessEventWithoutResValues = RawEventsSink.values.asScala.filter(ev => ev.mid().equals("ASSESS:12159f2827880221eef12a6be9560379:test1")).head
+    assessEventWithoutResValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (0)
+
+
+    val assessEventWithResValues = RawEventsSink.values.asScala.filter(ev => ev.mid().equals("ASSESS:12159f2827880221eef12a6be9560379:test2")).head
+    assessEventWithResValues.getTelemetry.read[util.Map[String, AnyRef]]("edata").get.get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (1)
+
+    /*
     val responseEventWithoutValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(39)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
     responseEventWithoutValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("values").asInstanceOf[util.ArrayList[AnyRef]].size should be (0)
+
     val assessEventWithoutResValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(40)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
     assessEventWithoutResValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (0)
+
     val assessEventWithResValues = gson.fromJson(gson.toJson(RawEventsSink.values.get(42)), new util.LinkedHashMap[String, AnyRef]().getClass).asInstanceOf[util.Map[String, AnyRef]].asScala
     assessEventWithResValues("edata").asInstanceOf[util.Map[String, AnyRef]].get("resvalues").asInstanceOf[util.ArrayList[AnyRef]].size should be (1)
+    */
 
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successBatchCount}").getValue() should be (3)
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.failedBatchCount}").getValue() should be (1)
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.failedEventCount}").getValue() should be (2)
-    BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successEventCount}").getValue() should be (43)
+    BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.successEventCount}").getValue() should be (44)
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.unique-event-count").getValue() should be (2)
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.duplicate-event-count").getValue() should be (1)
     BaseMetricsReporter.gaugeMetrics(s"${extractorConfig.jobName}.${extractorConfig.auditEventCount}").getValue() should be (2)
@@ -134,13 +156,14 @@ class RawEventsSink extends SinkFunction[util.Map[String, AnyRef]] {
 
   override def invoke(value: util.Map[String, AnyRef]): Unit = {
     synchronized {
-      RawEventsSink.values.add(value)
+      val res = new Event(value.asInstanceOf[util.Map[String, Any]])
+      RawEventsSink.values.add(new Event(value.asInstanceOf[util.Map[String, Any]]))
     }
   }
 }
 
 object RawEventsSink {
-  val values: util.List[util.Map[String, AnyRef]] = new util.ArrayList()
+  val values: util.List[Event] = new util.ArrayList[Event]()
 }
 
 class FailedEventsSink extends SinkFunction[util.Map[String, AnyRef]] {
@@ -172,13 +195,26 @@ class LogEventsSink extends SinkFunction[util.Map[String, AnyRef]] {
 
   override def invoke(value: util.Map[String, AnyRef]): Unit = {
     synchronized {
-      LogEventsSink.values.add(value)
+      LogEventsSink.values.add(new Event(value.asInstanceOf[util.Map[String, Any]]))
     }
   }
 }
 
 object LogEventsSink {
-  val values: util.List[util.Map[String, AnyRef]] = new util.ArrayList()
+  val values: util.List[Event] = new util.ArrayList[Event]()
+}
+
+
+class AuditEventsSink extends SinkFunction[String] {
+  override def invoke(value: String): Unit = {
+    synchronized {
+      AuditEventsSink.values.add(value)
+    }
+  }
+}
+
+object AuditEventsSink {
+  val values: util.List[String] = new util.ArrayList()
 }
 
 class DupEventsSink extends SinkFunction[util.Map[String, AnyRef]] {
diff --git a/data-pipeline-flink/user-cache-updater/README.md b/data-pipeline-flink/user-cache-updater-2.0/README.md
similarity index 100%
rename from data-pipeline-flink/user-cache-updater/README.md
rename to data-pipeline-flink/user-cache-updater-2.0/README.md
diff --git a/data-pipeline-flink/user-cache-updater-2.0/pom.xml b/data-pipeline-flink/user-cache-updater-2.0/pom.xml
new file mode 100644
index 0000000000..d976893de8
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/pom.xml
@@ -0,0 +1,240 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+    <prerequisites>
+        <maven>3.0.1</maven>
+    </prerequisites>
+
+    <parent>
+        <groupId>org.sunbird.dp</groupId>
+        <artifactId>pipeline-jobs</artifactId>
+        <version>1.0</version>
+    </parent>
+
+    <groupId>org.sunbird.dp.jobs</groupId>
+    <artifactId>user-cache-updater-2.0</artifactId>
+    <version>2.0.0</version>
+    <packaging>jar</packaging>
+    <name>UserCacheUpdaterV2</name>
+    <description>
+        Updating user details into readis cache
+    </description>
+
+    <properties>
+        <encoding>UTF-8</encoding>
+        <scoverage.plugin.version>1.4.0</scoverage.plugin.version>
+    </properties>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-scala_${scala.maj.version}</artifactId>
+            <version>${flink.version}</version>
+            <scope>provided</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.sunbird.dp</groupId>
+            <artifactId>dp-core</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+        <dependency>
+            <groupId>org.sunbird.dp</groupId>
+            <artifactId>dp-core</artifactId>
+            <version>1.0.0</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-test-utils_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-runtime_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+            <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+            <groupId>it.ozimov</groupId>
+            <artifactId>embedded-redis</artifactId>
+            <version>0.7.1</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.google.guava</groupId>
+                    <artifactId>guava</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>org.cassandraunit</groupId>
+            <artifactId>cassandra-unit</artifactId>
+            <version>3.11.2.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.flink</groupId>
+            <artifactId>flink-streaming-java_2.12</artifactId>
+            <version>${flink.version}</version>
+            <scope>test</scope>
+            <classifier>tests</classifier>
+        </dependency>
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.12</artifactId>
+            <version>3.0.6</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+            <version>3.3.3</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.fiftyonred</groupId>
+            <artifactId>mock-jedis</artifactId>
+            <version>0.4.0</version>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>mockwebserver</artifactId>
+            <version>4.4.0</version>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <sourceDirectory>src/main/scala</sourceDirectory>
+        <testSourceDirectory>src/test/scala</testSourceDirectory>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.1</version>
+                <configuration>
+                    <release>11</release>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <!-- Run shade goal on package phase -->
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <shadedArtifactAttached>false</shadedArtifactAttached>
+                            <artifactSet>
+                                <excludes>
+                                    <exclude>com.google.code.findbugs:jsr305</exclude>
+                                </excludes>
+                            </artifactSet>
+                            <filters>
+                                <filter>
+                                    <!-- Do not copy the signatures in the META-INF folder.
+                                    Otherwise, this might cause SecurityExceptions when using the JAR. -->
+                                    <artifact>*:*</artifact>
+                                    <excludes>
+                                        <exclude>META-INF/*.SF</exclude>
+                                        <exclude>META-INF/*.DSA</exclude>
+                                        <exclude>META-INF/*.RSA</exclude>
+                                    </excludes>
+                                </filter>
+                            </filters>
+                            <transformers>
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTaskV2</mainClass>
+                                </transformer>
+                                <!-- append default configs -->
+                                <transformer
+                                        implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+                                    <resource>reference.conf</resource>
+                                </transformer>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <groupId>net.alchim31.maven</groupId>
+                <artifactId>scala-maven-plugin</artifactId>
+                <version>4.4.0</version>
+                <configuration>
+                    <source>${java.target.runtime}</source>
+                    <target>${java.target.runtime}</target>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                    <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>scala-compile-first</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>add-source</goal>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>scala-test-compile</id>
+                        <phase>process-test-resources</phase>
+                        <goals>
+                            <goal>testCompile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+
+            <plugin>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>2.22.2</version>
+                <configuration>
+                    <skipTests>true</skipTests>
+                </configuration>
+            </plugin>
+
+            <plugin>
+                <groupId>org.scalatest</groupId>
+                <artifactId>scalatest-maven-plugin</artifactId>
+                <version>1.0</version>
+                <configuration>
+                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+                    <junitxml>.</junitxml>
+                    <filereports>usercache-testsuite.txt</filereports>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>test</id>
+                        <goals>
+                            <goal>test</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.scoverage</groupId>
+                <artifactId>scoverage-maven-plugin</artifactId>
+                <version>${scoverage.plugin.version}</version>
+                <configuration>
+                    <scalaVersion>${scala.version}</scalaVersion>
+                    <aggregate>true</aggregate>
+                    <highlighting>true</highlighting>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf b/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf
new file mode 100644
index 0000000000..524cdcde83
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/resources/user-cache-updater-v2.conf
@@ -0,0 +1,32 @@
+include "base-config.conf"
+
+kafka {
+  input.topic = ${job.env}".telemetry.audit"
+  groupId = ${job.env}"-user-cache-updater-group"
+}
+
+task {
+  usercache.updater.parallelism = 1
+}
+
+# redis-metadata
+redis-meta {
+  database {
+    userstore.id = 12
+    key.expiry.seconds = 3600
+  }
+}
+
+user-read {
+  api {
+    url = "/learner/private/user/v1/read"
+  }
+}
+
+regd.user.producer.pid = "learner-service"
+user.self.signin.types = ["google","self"]
+user.validated.types = ["sso"]
+user.self.signin.key = "Self-Signed-In"
+user.valid.key = "Validated"
+user.read.url.fields = "locations,organisations"
+user.read.api.error = ["CLIENT_ERROR"]
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala
new file mode 100644
index 0000000000..2dbd4dd28b
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala
@@ -0,0 +1,42 @@
+package org.sunbird.dp.usercache.domain
+
+import java.util
+
+import org.sunbird.dp.core.domain.Events
+import org.sunbird.dp.usercache.util.UserReadResult
+
+class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
+
+  override def kafkaKey(): String = {
+    did()
+  }
+
+  def getId: String = {
+    Option(objectType()).map({ t => if (t.equalsIgnoreCase("User")) objectID() else null
+    }).getOrElse(null)
+  }
+
+  def getState: String = {
+    telemetry.read[String]("edata.state").getOrElse(null)
+  }
+
+  def getContextDataId(cDataType: String): String = {
+    val cdata = telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("context.cdata").getOrElse(null)
+    var signInType: String = null
+    Option(cdata).map(data => {
+      data.forEach(cdataMap => {
+        if (cdataMap.get("type").asInstanceOf[String].equalsIgnoreCase(cDataType)) signInType = cdataMap.get("id").toString else signInType
+      })
+    }).getOrElse(signInType)
+    signInType
+  }
+
+  def userMetaData(): util.ArrayList[String] = {
+    telemetry.read[util.ArrayList[String]]("edata.props").getOrElse(new util.ArrayList[String]())
+  }
+
+  def isValid(userReadRes: UserReadResult) = {
+    if (userReadRes.responseCode.toUpperCase.equalsIgnoreCase("OK") && !userReadRes.result.isEmpty && userReadRes.result.containsKey("response")) true else false
+  }
+
+}
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala
new file mode 100644
index 0000000000..3818fd7e2e
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunctionV2.scala
@@ -0,0 +1,80 @@
+package org.sunbird.dp.usercache.functions
+
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.configuration.Configuration
+import org.apache.flink.streaming.api.functions.ProcessFunction
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.contentupdater.core.util.RestUtil
+import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
+import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
+import org.sunbird.dp.core.util.JSONUtil
+import org.sunbird.dp.usercache.domain.Event
+import org.sunbird.dp.usercache.task.UserCacheUpdaterConfigV2
+import org.sunbird.dp.usercache.util.UserMetadataUpdater
+
+import scala.collection.JavaConverters.mapAsJavaMap
+import scala.collection.mutable
+
+class UserCacheUpdaterFunctionV2(config: UserCacheUpdaterConfigV2)(implicit val mapTypeInfo: TypeInformation[Event])
+  extends BaseProcessFunction[Event, Event](config) {
+
+  private[this] val logger = LoggerFactory.getLogger(classOf[UserCacheUpdaterFunctionV2])
+  private var dataCache: DataCache = _
+
+  private var restUtil: RestUtil = _
+
+  override def metricsList(): List[String] = {
+    List(config.userCacheHit, config.skipCount, config.successCount, config.totalEventsCount, config.apiReadMissCount, config.apiReadSuccessCount)
+  }
+
+  override def open(parameters: Configuration): Unit = {
+    super.open(parameters)
+    dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.userStore, config.userFields)
+    dataCache.init()
+    restUtil = new RestUtil()
+  }
+
+  override def close(): Unit = {
+    super.close()
+    dataCache.close()
+  }
+
+  override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = {
+    metrics.incCounter(config.totalEventsCount)
+    val userId = event.getId
+    try {
+      Option(userId).map(id => {
+        Option(event.getState).map(name => {
+          val userData: mutable.Map[String, AnyRef] = name.toUpperCase match {
+            case "CREATE" | "CREATED" | "UPDATE" | "UPDATED" => {
+              UserMetadataUpdater.execute(id, event, metrics, config, dataCache, restUtil)
+            }
+            case _ => {
+              logger.info(s"Invalid event state name either it should be(Create/Created/Update/Updated) but found $name for ${event.mid()}")
+              metrics.incCounter(config.skipCount)
+              mutable.Map[String, AnyRef]()
+            }
+          }
+          if (!userData.isEmpty) {
+            if (config.regdUserProducerPid.equals(event.producerPid())) UserMetadataUpdater.removeEmptyFields(config.userStoreKeyPrefix + id, dataCache, userData)
+            dataCache.hmSet(config.userStoreKeyPrefix + id, mapAsJavaMap(UserMetadataUpdater.stringify(userData)))
+            logger.info(s"Data inserted into cache for user: ${userId} having mid: ${event.mid()}")
+            metrics.incCounter(config.successCount)
+            metrics.incCounter(config.userCacheHit)
+          } else {
+            logger.info(s"User Data to be updated is empty for user: ${userId}")
+            metrics.incCounter(config.skipCount)
+          }
+        }).getOrElse(metrics.incCounter(config.skipCount))
+      }).getOrElse(metrics.incCounter(config.skipCount))
+    } catch {
+      case ex: Exception => {
+        ex.printStackTrace()
+        logger.info(s"Processing event for user: ${userId} having mid: ${event.mid()}")
+        logger.info("Event throwing exception: ", JSONUtil.serialize(event))
+        throw ex
+      }
+    }
+  }
+}
+
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala
new file mode 100644
index 0000000000..57a2f0931a
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfigV2.scala
@@ -0,0 +1,82 @@
+package org.sunbird.dp.usercache.task
+
+import com.typesafe.config.Config
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.sunbird.dp.core.job.BaseJobConfig
+import org.sunbird.dp.usercache.domain.Event
+
+import java.util.{List => JList}
+
+class UserCacheUpdaterConfigV2(override val config: Config) extends BaseJobConfig(config, "UserCacheUpdaterJobV2") {
+
+  private val serialVersionUID = 2905979434303791379L
+
+  implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
+
+  // Kafka Topics Configuration
+  val inputTopic: String = config.getString("kafka.input.topic")
+  val userFields: List[String] = if (config.hasPath("user.redis.removeable-fields"))
+    config.getStringList("user.redis.removeable-fields").asInstanceOf[List[String]]
+  else List[String]("state", "district", "block", "cluster", "schooludisecode", "schoolname")
+
+  // User cache updater job metrics
+  val userCacheHit = "user-cache-hit"
+  val skipCount = "skipped-message-count"
+  val successCount = "success-message-count"
+  val dbReadSuccessCount = "db-read-success-count"
+  val dbReadMissCount = "db-read-miss-count"
+  val apiReadSuccessCount = "api-read-success-count"
+  val apiReadMissCount = "api-read-miss-count"
+  val totalEventsCount ="total-audit-events-count"
+
+  val userSelfSignedInTypeList: JList[String] = config.getStringList("user.self.signin.types")
+  val userValidatedTypeList: JList[String] = config.getStringList("user.validated.types")
+  val userSelfSignedKey: String = config.getString("user.self.signin.key")
+  val userValidatedKey: String = config.getString("user.valid.key")
+  val regdUserProducerPid: String = config.getString("regd.user.producer.pid")
+
+  // Redis
+  val userStore: Int = config.getInt("redis-meta.database.userstore.id")
+
+  val userCacheParallelism: Int = config.getInt("task.usercache.updater.parallelism")
+
+  // constants
+  val userSignInTypeKey = "usersignintype"
+  val userLoginTypeKey = "userlogintype"
+  val firstName = "firstname"
+  val lastName = "lastname"
+  val rootOrgId = "rootorgid"
+  val stateKey = "state"
+  val districtKey = "district"
+  val blockKey = "block"
+  val clusterKey = "cluster"
+  val orgnameKey = "orgname"
+  val schoolKey = "school"
+  val schoolUdiseCodeKey = "schooludisecode"
+  val schoolNameKey = "schoolname"
+  val `type` = "type"
+  val subtype = "subType"
+  val userTypeKey = "usertype"
+  val userSubtypeKey = "usersubtype"
+  val userId = "userid"
+  val language = "language"
+  val email = "email"
+  val phone = "phone"
+  val profileUserTypesKey = "profileusertypes"
+
+  //user store key prefix
+  val userStoreKeyPrefix = "user:"
+  // Consumers
+  val userCacheConsumer = "user-cache-consumer"
+
+  // Functions
+  val userCacheUpdaterFunction = "UserCacheUpdaterFunctionV2"
+
+  //User Read API
+  val userReadApiUrl = config.getString("user-read.api.url")
+  val userReadApiFields = config.getString("user.read.url.fields")
+  val userReadApiErrors: JList[String] = config.getStringList("user.read.api.error")
+
+  val userAccBlockedErrCode = "UOS_USRRED0006"
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala
new file mode 100644
index 0000000000..dc99cf4ebc
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTaskV2.scala
@@ -0,0 +1,48 @@
+package org.sunbird.dp.usercache.task
+
+import java.io.File
+
+import com.typesafe.config.ConfigFactory
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.api.java.utils.ParameterTool
+import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
+import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.core.util.FlinkUtil
+import org.sunbird.dp.usercache.domain.Event
+import org.sunbird.dp.usercache.functions.UserCacheUpdaterFunctionV2
+
+class UserCacheUpdaterStreamTaskV2(config: UserCacheUpdaterConfigV2, kafkaConnector: FlinkKafkaConnector) {
+
+  private val serialVersionUID = -7729362727131516112L
+
+  def process(): Unit = {
+
+    implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config)
+    implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
+
+    val source = kafkaConnector.kafkaEventSource[Event](config.inputTopic)
+    env.addSource(source, config.userCacheConsumer).uid(config.userCacheConsumer).rebalance()
+      .process(new UserCacheUpdaterFunctionV2(config)).setParallelism(config.userCacheParallelism)
+      .name(config.userCacheUpdaterFunction).uid(config.userCacheUpdaterFunction)
+    env.execute(config.jobName)
+  }
+
+}
+
+// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster
+object UserCacheUpdaterStreamTaskV2 {
+
+  def main(args: Array[String]): Unit = {
+    val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path"))
+    val config = configFilePath.map {
+      path => ConfigFactory.parseFile(new File(path)).resolve()
+    }.getOrElse(ConfigFactory.load("user-cache-updater-v2.conf").withFallback(ConfigFactory.systemEnvironment()))
+    val userCacheUpdaterConfig = new UserCacheUpdaterConfigV2(config)
+    val kafkaUtil = new FlinkKafkaConnector(userCacheUpdaterConfig)
+    val task = new UserCacheUpdaterStreamTaskV2(userCacheUpdaterConfig, kafkaUtil)
+    task.process()
+  }
+}
+
+// $COVERAGE-ON$
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala
new file mode 100644
index 0000000000..f035891040
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/main/scala/org/sunbird/dp/usercache/util/UserMetadataUpdater.scala
@@ -0,0 +1,153 @@
+package org.sunbird.dp.usercache.util
+
+import com.google.gson.Gson
+import org.slf4j.LoggerFactory
+import org.sunbird.dp.contentupdater.core.util.RestUtil
+import org.sunbird.dp.core.cache.DataCache
+import org.sunbird.dp.core.job.Metrics
+import org.sunbird.dp.usercache.domain.Event
+import org.sunbird.dp.usercache.task.UserCacheUpdaterConfigV2
+
+import scala.collection.JavaConverters._
+import scala.collection.mutable
+
+case class UserReadResult(result: java.util.HashMap[String, Any], responseCode: String, params: Params)
+case class Response(firstName: String, lastName: String, encEmail: String, encPhone: String, language: java.util.List[String], rootOrgId: String, profileUserType: java.util.HashMap[String, String],
+                    userLocations: java.util.ArrayList[java.util.Map[String, AnyRef]], rootOrg: RootOrgInfo, userId: String, framework: java.util.LinkedHashMap[String, java.util.List[String]], profileUserTypes: java.util.List[java.util.HashMap[String, String]])
+case class RootOrgInfo(orgName: String)
+case class Params(msgid: String, err: String, status: String, errmsg: String)
+
+object UserMetadataUpdater {
+
+  private lazy val gson = new Gson()
+
+  val logger = LoggerFactory.getLogger("UserMetadataUpdater")
+
+  def execute(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache, restUtil: RestUtil): mutable.Map[String, AnyRef] = {
+
+    val generalInfo = getGeneralInfo(userId, event, metrics, config, dataCache);
+    val regdInfo = if (config.regdUserProducerPid.equals(event.producerPid())) {
+      getRegisteredUserInfo(userId, event, metrics, config, dataCache, restUtil)
+    } else mutable.Map[String, String]()
+    generalInfo.++:(regdInfo);
+  }
+
+  def getGeneralInfo(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache): mutable.Map[String, String] = {
+    val userCacheData: mutable.Map[String, String] = mutable.Map[String, String]()
+    Option(event.getContextDataId(cDataType = "SignupType")).map(signInType => {
+      if (config.userSelfSignedInTypeList.contains(signInType)) {
+        userCacheData.put(config.userSignInTypeKey, config.userSelfSignedKey)
+      }
+      if (config.userValidatedTypeList.contains(signInType)) {
+        userCacheData.put(config.userSignInTypeKey, config.userValidatedKey)
+      }
+    }).orNull
+    Option(event.getContextDataId(cDataType = "UserRole")).map(loginType => {
+      userCacheData.put(config.userLoginTypeKey, loginType)
+    })
+    userCacheData;
+  }
+
+  @throws(classOf[Exception])
+  def getRegisteredUserInfo(userId: String, event: Event, metrics: Metrics, config: UserCacheUpdaterConfigV2, dataCache: DataCache,
+                            restUtil: RestUtil): mutable.Map[String, AnyRef] = {
+    var userCacheData: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]()
+
+    //?fields=locations is appended in url to get userLocation in API response
+    val userReadRes = gson.fromJson[UserReadResult](restUtil.get(String.format("%s%s",config.userReadApiUrl, userId + "?fields=" + config.userReadApiFields)), classOf[UserReadResult])
+    if(event.isValid(userReadRes)) {
+      // Inc API Read metrics
+      metrics.incCounter(config.apiReadSuccessCount)
+
+      val response = gson.fromJson[Response](gson.toJson(userReadRes.result.get("response")), classOf[Response])
+      val framework = response.framework
+      //flatten BGMS value
+      /**
+        * Assumption: Board and Framework-id is single valued
+        */
+      if (!framework.isEmpty) {
+        val boardList = framework.getOrDefault("board", List().asJava)
+        val board = if (!boardList.isEmpty) boardList.get(0) else ""
+        val medium = framework.getOrDefault("medium", List().asJava)
+        val grade = framework.getOrDefault("gradeLevel", List().asJava)
+        val subject = framework.getOrDefault("subject", List().asJava)
+        val frameworkIdList = framework.getOrDefault("id", List().asJava)
+        val id = if (!frameworkIdList.isEmpty) frameworkIdList.get(0) else ""
+        userCacheData.+=("board" -> board, "medium" -> medium, "grade" -> grade, "subject" -> subject, "framework" -> id)
+      }
+
+      //Location and School Information
+      val locationInfo = response.userLocations
+      if(null != locationInfo && !locationInfo.isEmpty) {
+        locationInfo.forEach(location => {
+          location.getOrDefault("type", "").asInstanceOf[String].toLowerCase match {
+          case config.schoolKey => userCacheData.put(config.schoolNameKey, location.getOrDefault("name", "").asInstanceOf[String])
+              userCacheData.put(config.schoolUdiseCodeKey, location.getOrDefault("code", "").asInstanceOf[String])
+          case _ => userCacheData.put(location.getOrDefault("type", "").asInstanceOf[String], location.getOrDefault("name", "").asInstanceOf[String])
+          }
+        })
+      }
+
+      //Flatten User Type and subType
+      val profileUserTypes = response.profileUserTypes
+      if (null != profileUserTypes && !profileUserTypes.isEmpty) {
+        val List(userTypeString, userSubtypeString) = makeUsertypeStrings(profileUserTypes, config)
+
+        userCacheData.+=(config.userTypeKey -> userTypeString, config.userSubtypeKey -> userSubtypeString)
+        userCacheData.+=(config.profileUserTypesKey -> new Gson().toJson(profileUserTypes))
+      }
+
+      //Personal information
+      userCacheData.+=(config.firstName -> response.firstName, config.lastName -> response.lastName,
+        config.language -> response.language,
+        config.orgnameKey -> response.rootOrg.orgName,
+        config.rootOrgId -> response.rootOrgId,
+        config.phone -> response.encPhone,
+        config.email -> response.encEmail,
+        config.userId -> response.userId)
+
+    } else if (config.userReadApiErrors.contains(userReadRes.responseCode.toUpperCase) && userReadRes.params.err.equalsIgnoreCase(config.userAccBlockedErrCode)) { //Skip the events for which response is 400 Bad request
+      logger.info(s"User Read API has response as ${userReadRes.responseCode.toUpperCase} for user: ${userId}")
+      metrics.incCounter(config.apiReadMissCount)
+    } else {
+      logger.info(s"User Read API does not have details for user: ${userId}")
+      metrics.incCounter(config.apiReadMissCount)
+      throw new Exception(s"User Read API does not have details for user: ${userId}")
+    }
+    userCacheData
+  }
+
+  def makeUsertypeStrings(profileUserTypes: java.util.List[java.util.HashMap[String, String]], config: UserCacheUpdaterConfigV2): List[String] = {
+    val userTypeValue = mutable.ListBuffer[String]()
+    val userSubtypeValue = mutable.ListBuffer[String]()
+    profileUserTypes.forEach(userType => {
+      val typeVal:String = userType.get(config.`type`)
+      val subTypeVal:String = userType.get(config.subtype)
+
+      if (typeVal != null && typeVal.nonEmpty && !userTypeValue.contains(typeVal)) userTypeValue.append(typeVal)
+      if (subTypeVal != null && subTypeVal.nonEmpty && !userSubtypeValue.contains(subTypeVal)) userSubtypeValue.append(subTypeVal)
+    })
+
+    List(userTypeValue.mkString(","), userSubtypeValue.mkString(","))
+  }
+
+  def removeEmptyFields(key: String, dataCache: DataCache, userMetaData: mutable.Map[String, AnyRef]):Unit = {
+    val redisRec = dataCache.hgetAllWithRetry(key)
+    val removableKeys = redisRec.keySet.diff(userMetaData.keySet)
+    if(removableKeys.nonEmpty) dataCache.hdelWithRetry(key, removableKeys.toSeq)
+  }
+
+  def stringify(userData: mutable.Map[String, AnyRef]): mutable.Map[String, String] = {
+    userData.map { f =>
+      (f._1, if (!f._2.isInstanceOf[String]) {
+        if (null != f._2) {
+          new Gson().toJson(f._2)
+        } else {
+          ""
+        }
+      } else {
+        f._2.asInstanceOf[String].replaceAll("\\[", "\\\\[").replaceAll("\\]", "\\\\]")
+      })
+    }
+  }
+}
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf b/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf
new file mode 100644
index 0000000000..3b1ac222e6
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/resources/test.conf
@@ -0,0 +1,31 @@
+include "base-test.conf"
+
+kafka {
+  input.topic = "flink.telemetry.audit"
+  groupId = "flink-user-cache-updater-group"
+}
+
+task {
+  usercache.updater.parallelism = 1
+}
+
+redis-meta {
+  database {
+    userstore.id = 1
+    key.expiry.seconds = 3600
+  }
+}
+
+user-read {
+  api {
+    url = "http://127.0.0.1:3000/learner/private/user/v1/read/"
+  }
+}
+
+regd.user.producer.pid = "learner-service"
+user.self.signin.types = ["google","self"]
+user.validated.types = ["sso"]
+user.self.signin.key = "Self-Signed-In"
+user.valid.key = "Validated"
+user.read.url.fields = "locations,organisations"
+user.read.api.error = ["CLIENT_ERROR"]
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
new file mode 100644
index 0000000000..e25223e54b
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
@@ -0,0 +1,106 @@
+package org.sunbird.dp.fixture
+
+object EventFixture {
+
+  val telemetrEvents: List[String] = List(
+
+    /**
+      * User-Id : user-1
+      * Edata.state is update
+      * Having all the user data
+      */
+    """
+      {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-1","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin,
+    /**
+      * UserId: user-9
+      * Client_Error
+      */
+    """{"actor":{"id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0","type":"User"},"eid":"AUDIT","edata":{"state":"Update","type":"mergeUser","props":["fromAccountId","toAccountId","type"]},"ver":"3.0","syncts":1621593310099,"@timestamp":"2021-05-21T10:35:10.099Z","ets":1621593307552,"context":{"channel":"0126796199493140480","pdata":{"id":"preprod.diksha.learning.service","pid":"learner-service","ver":"3.9.0"},"env":"User","cdata":[{"id":"9e6f3392-af99-425b-9404-e8e4078a1b65","type":"FromAccountId"},{"id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0","type":"ToAccountId"},{"id":"8f0d93fd-33ac-4a64-a39c-f008011c976a","type":"Request"}],"rollup":{"l1":"01275678925675724817"}},"flags":{"pp_duplicate_skipped":true,"pp_validation_processed":true},"mid":"8f0d93fd-33ac-4a64-a39c-f008011c976a","type":"events","object":{"id":"user-8","type":"User"}}""",
+    /**
+      * User-Id : user-2
+      * Edata.state is update
+      * user's data missed from API
+      */
+    """
+      {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-2","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin,
+    /**
+      * User-Id : user-3
+      * Edata.state is update
+      * framework and block info not present
+      */
+    """
+      {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-3","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin,
+    /**
+      * User-Id : user-4
+      * Edata.state is update
+      * framework and block info not present
+      */
+    """
+      {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin,
+    /**
+      * UserId = user-5
+      * EData state is "Created"
+      * User SignupType is "sso"
+      * It should able to insert The Map(usersignintype, Validated)
+      *
+      */
+        """
+          |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-5"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin,
+
+    /**
+      * AUDIT Event with object type as content. (not related to user)
+      * Props are not defined - Skip Count should incr (skipCount =+ 1)
+      */
+    """
+      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"do_r8r8rew97we9r8","type":"content"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin,
+
+    /**
+      * AUDIT Event, But having invalid state(other than create/update)
+      * userid= user-6
+      */
+
+    """
+      |
+      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-7","type":"user"},"edata":{"state":"wrongState"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |""".stripMargin,
+    /**
+     * User-Id : user-10
+     * schoolname starting with '['
+     * Having all the user data
+     */
+    """
+      {"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-10","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |
+      |""".stripMargin
+
+  )
+
+  /**
+   * User-Id : user-1
+   * Edata.state is update
+   * Having pid as sunbird.app
+   */
+  val telemetryEventWithAppPid: String = """
+      |{"actor":{"type":"System","id":"cb0defc7-9f5f-4c0a-bb73-207d34ed2cf0"},"eid":"AUDIT","edata":{"state":"Updated","props":[]},"ver":"3.0","syncts":1631551986826,"@timestamp":"2021-09-13T16:53:06.826Z","ets":1.63155195956E12,"context":{"cdata":[{"id":"teacher","type":"UserRole"},{"type":"Tabs","id":"Library-Course"},{"id":"a9d1d5df-da2e-4006-8495-363cce1ceec2","type":"UserSession"}],"env":"sdk","channel":"0126684405014528002","pdata":{"id":"dev.sunbird.app","pid":"sunbird.app","ver":"4.1.892"},"sid":"a9d1d5df-da2e-4006-8495-363cce1ceec2","did":"575fc6d8a853d2f543e4ef54d57aa3b6cd0110cf","rollup":{"l1":"0126684405014528002"}},"flags":{"ex_processed":true,"pp_validation_processed":true,"pp_duplicate_skipped":true},"mid":"bb429169-f320-4da3-80ad-660093aafe3b","type":"events","object":{"id":"user-1","type":"user","version":"","rollup":{}}}
+      |""".stripMargin
+
+  /**
+   * User-Id : user-1
+   * Edata.state is update
+   * Having pid as learner-service
+   */
+  val telemetryEventWithLearnerPid: String = """
+      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"google","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-1","type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
+      |""".stripMargin
+
+}
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala
new file mode 100644
index 0000000000..e411a703f4
--- /dev/null
+++ b/data-pipeline-flink/user-cache-updater-2.0/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpecV2.scala
@@ -0,0 +1,348 @@
+package org.sunbird.dp.spec
+
+import java.io.IOException
+import java.util
+import com.google.gson.Gson
+import com.typesafe.config.{Config, ConfigFactory}
+import okhttp3.mockwebserver.{MockResponse, MockWebServer}
+import org.apache.flink.api.common.typeinfo.TypeInformation
+import org.apache.flink.api.java.typeutils.TypeExtractor
+import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
+import org.apache.flink.streaming.api.functions.source.SourceFunction
+import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
+import org.apache.flink.test.util.MiniClusterWithClientResource
+import org.mockito.Mockito
+import org.mockito.Mockito._
+import org.scalatest.BeforeAndAfterEach
+import org.sunbird.dp.core.cache.RedisConnect
+import org.sunbird.dp.core.job.FlinkKafkaConnector
+import org.sunbird.dp.fixture.EventFixture
+import org.sunbird.dp.usercache.domain.Event
+import org.sunbird.dp.usercache.task.{UserCacheUpdaterConfigV2, UserCacheUpdaterStreamTaskV2}
+import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
+import redis.clients.jedis.Jedis
+import redis.embedded.RedisServer
+
+class UserCacheUpdatetStreamTaskSpecV2 extends BaseTestSpec with BeforeAndAfterEach {
+
+  implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
+
+  val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
+    .setConfiguration(testConfiguration())
+    .setNumberSlotsPerTaskManager(1)
+    .setNumberTaskManagers(1)
+    .build)
+
+  var redisServer: RedisServer = _
+  val config: Config = ConfigFactory.load("test.conf")
+  val userCacheConfig: UserCacheUpdaterConfigV2 = new UserCacheUpdaterConfigV2(config)
+  val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
+  val gson = new Gson()
+  var jedis: Jedis = _
+  var server = new MockWebServer()
+
+  override protected def beforeAll(): Unit = {
+    super.beforeAll()
+    redisServer = new RedisServer(6340)
+    redisServer.start()
+    BaseMetricsReporter.gaugeMetrics.clear()
+    val redisConnect = new RedisConnect(userCacheConfig.metaRedisHost, userCacheConfig.metaRedisPort, userCacheConfig)
+    jedis = redisConnect.getConnection(userCacheConfig.userStore)
+    flinkCluster.before()
+  }
+
+  override protected def afterAll(): Unit = {
+    super.afterAll()
+    redisServer.stop()
+    server.close()
+    flinkCluster.after()
+  }
+
+  override protected def beforeEach(): Unit = {
+    server.close()
+    server = new MockWebServer()
+    super.beforeEach()
+  }
+
+  def setupRestUtilData(): Unit = {
+
+    val user1 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"hm"},{"type":"teacher"},{"type":"administrator","subType":"crp"},{"type":"other"},{"type":"parent"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val clientError = """{"id":".private.user.v1.read.bcda3d80-0de0-4179-a702-9f89e6eeec56","ver":"private","ts":"2021-05-24 07:10:48:908+0000","params":{"resmsgid":null,"msgid":"f17cf782-6f67-4575-9152-21cf2daabe85","err":"USER_ACCOUNT_BLOCKED","status":"USER_ACCOUNT_BLOCKED","errmsg":"User account has been blocked ."},"responseCode":"CLIENT_ERROR","result":{}}"""
+    val user2 = """{"id":".private.user.v1.read.95e4942d-cbe8-477d-aebd-ad8e6de4bfc81","ver":"private","ts":"2021-03-16 11:42:24:074+0000","params":{"resmsgid":null,"msgid":"e082153a-159c-4d18-a2d3-a5dc509f4d9f","err":"USER_NOT_FOUND","status":"USER_NOT_FOUND","errmsg":"user not found."},"responseCode":"RESOURCE_NOT_FOUND","result":{}}"""
+    val user3 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":[],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"user-3","recoveryEmail":"","identifier":"user-3","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"01285019302823526477","prevUsedEmail":"","firstName":"Isha","lastName":"Wakankar","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"user-3","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"deo"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"user-3","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val user4 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"thumbnail":null,"encPhone": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==", "encEmail": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==","updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Manju","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Davanam","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"teacher"},"profileUserTypes":[{"type":"teacher"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"type":"location","name":""}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":[],"gradeLevel":["Volunteers"],"medium":["English"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val invalid_json = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-04-21T02:51:39ZZ\",\"params\":{\"resmsgid\":\"4544fce4-efee-4ee2-8816-fdb3f60ac492\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"status\":\"No dialcodeFound\"}}"
+    val user10 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator-hm","subType":null},"profileUserTypes":[{"type":"administrator","subType":null}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1]","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"[RPMMAT M.S UDHADIH","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    try
+      server.start(3000)
+    catch {
+      case e: IOException =>
+        System.out.println("Exception" + e)
+    }
+    server.enqueue(new MockResponse().setBody(user1))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-1")
+    server.enqueue(new MockResponse().setBody(user1))
+//    server.enqueue(new MockResponse().setBody(clientError))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-9")
+    server.enqueue(new MockResponse().setBody(user1))
+//    server.enqueue(new MockResponse().setBody(user2))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-2")
+    server.enqueue(new MockResponse().setBody(user3))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-3")
+    server.enqueue(new MockResponse().setBody(user4))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-4")
+    server.enqueue(new MockResponse().setBody(user1))
+//    server.enqueue(new MockResponse().setBody(user2))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-5")
+    server.enqueue(new MockResponse().setBody(user10))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-10")
+//    server.enqueue(new MockResponse().setBody(invalid_json))
+//    server.url("http://127.0.0.1:3000/api/dialcode/v3/read/X3J6W2")
+//    server.enqueue(new MockResponse().setBody(invalid_json))
+//    server.url("http://127.0.0.1:3000/api/dialcode/v3/read/X3J6W3")
+    server.enqueue(new MockResponse().setHeader("Authorization", "auth_token"))
+
+  }
+
+  def setupRestUtilDataWithErrors(): Unit = {
+
+    val user1 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":["English"],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","recoveryEmail":"","identifier":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Utkarsha","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Kapoor","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"administrator","subType":"deo"},"profileUserTypes":[{"type":"administrator","subType":"hm"},{"type":"teacher"},{"type":"administrator","subType":"crp"},{"type":"other"},{"type":"parent"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":["IGOT-Health"],"gradeLevel":["Volunteers"],"id":["igot_health"],"medium":["English"],"subject":["IRCS"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val clientError = """{"id":".private.user.v1.read.bcda3d80-0de0-4179-a702-9f89e6eeec56","ver":"private","ts":"2022-03-17 08:33:32:188+0000","params":{"resmsgid":"4896831f-ef38-4731-bb0b-564a05e98576","msgid":"4896831f-ef38-4731-bb0b-564a05e98576","err":"UOS_USRRED0006","status":"FAILED","errmsg":"User account has been blocked ."},"responseCode":"CLIENT_ERROR","result":{}}"""
+    val user2 = """{"id":".private.user.v1.read.95e4942d-cbe8-477d-aebd-ad8e6de4bfc81","ver":"private","ts":"2021-03-16 11:42:24:074+0000","params":{"resmsgid":null,"msgid":"e082153a-159c-4d18-a2d3-a5dc509f4d9f","err":"USER_NOT_FOUND","status":"USER_NOT_FOUND","errmsg":"user not found."},"responseCode":"RESOURCE_NOT_FOUND","result":{}}"""
+    val user3 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"webPages":[],"maskedPhone":null,"subject":[],"channel":"root2","language":[],"updatedDate":null,"password":null,"managedBy":null,"flagsValue":2,"id":"user-3","recoveryEmail":"","identifier":"user-3","thumbnail":null,"updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"01285019302823526477","prevUsedEmail":"","firstName":"Isha","lastName":"Wakankar","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"user-3","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserTypes":[{"type":"administrator","subType":"deo"}],"profileUserType":{"type":"administrator","subType":"deo"},"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"name":"DPS, MATHURA","id":"63087424e-18cb-49b0-865c-98f265c73ed3","type":"school","code":"3183211"}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"user-3","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val user4 = """{"id":".private.user.v1.read.123456","ver":"private","ts":"2021-03-09 11:33:42:061+0530","params":{"resmsgid":null,"msgid":"06ff91d6-043e-4714-b002-6f8b90a04723","err":null,"status":"success","errmsg":null},"responseCode":"OK","result":{"response":{"thumbnail":null,"encPhone": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==", "encEmail": "LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==","updatedBy":null,"accesscode":null,"locationIds":["8db3345c-1bfc-4276-aef1-7ea0f3183211","d087424e-18cb-49b0-865c-98f265c73ed3","13087424e-18cb-49b0-865c-98f265c73ed3","43087424e-18cb-49b0-865c-98f265c73ed3"],"externalIds":[{"idType":"root2","provider":"root2","id":"123456"}],"registryId":null,"rootOrgId":"0127738024883077121","prevUsedEmail":"","firstName":"Manju","tncAcceptedOn":null,"allTncAccepted":{},"phone":"","dob":null,"grade":[],"currentLoginTime":null,"userType":null,"status":1,"lastName":"Davanam","tncLatestVersion":"v1","gender":null,"roles":["PUBLIC"],"prevUsedPhone":"","stateValidated":false,"isDeleted":false,"organisations":[{"organisationId":"0127738024883077121","updatedBy":null,"addedByName":null,"addedBy":null,"roles":["PUBLIC"],"approvedBy":null,"updatedDate":null,"userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","approvaldate":null,"isDeleted":false,"hashTagId":"0127738024883077121","isRejected":null,"id":"0132322953313320960","position":null,"isApproved":null,"orgjoindate":"2021-03-09 11:31:31:930+0530","orgLeftDate":null}],"profileUserType":{"type":"teacher"},"profileUserTypes":[{"type":"teacher"}],"userLocations":[{"code":"21","name":"Odisha","id":"8db3345c-1bfc-4276-aef1-7ea0f3183211","type":"state","parentId":null},{"code":"2112","name":"CUTTACK","id":"d087424e-18cb-49b0-865c-98f265c73ed3","type":"district","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"BLOCK1","id":"13087424e-18cb-49b0-865c-98f265c73ed3","type":"block","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"code":"211","name":"CLUSTER1","id":"43087424e-18cb-49b0-865c-98f265c73ed3","type":"cluster","parentId":"8db3345c-1bfc-4276-aef1-7ea0f3183211"},{"type":"location","name":""}],"countryCode":null,"tncLatestVersionUrl":"https://dev-sunbird-temp.azureedge.net/portal/terms-and-conditions-v1.html","maskedEmail":"am******@yopmail.com","tempPassword":null,"email":"am******@yopmail.com","rootOrg":{"dateTime":null,"preferredLanguage":"English","keys":{},"approvedBy":null,"channel":"root2","description":"Root Org2","updatedDate":null,"addressId":null,"orgType":null,"provider":null,"locationId":null,"orgCode":null,"theme":null,"id":"0127738024883077121","communityId":null,"isApproved":null,"email":null,"slug":"root2","isSSOEnabled":null,"thumbnail":null,"orgName":"Root Org2","updatedBy":null,"locationIds":[],"externalId":null,"isRootOrg":true,"rootOrgId":"0127738024883077121","approvedDate":null,"imgUrl":null,"homeUrl":null,"orgTypeId":null,"isDefault":null,"createdDate":"2019-05-31 16:41:29:485+0530","createdBy":null,"parentOrgId":null,"hashTagId":"0127738024883077121","noOfMembers":null,"status":1},"phoneVerified":false,"profileSummary":null,"recoveryPhone":"","avatar":null,"userName":"creatortest_72yz","userId":"a962a4ff-b5b5-46ad-a9fa-f54edf1bcccb","userSubType":null,"promptTnC":true,"emailVerified":true,"lastLoginTime":null,"createdDate":"2021-03-09 11:31:23:189+0530","framework":{"board":[],"gradeLevel":["Volunteers"],"medium":["English"]},"createdBy":null,"location":null,"tncAcceptedVersion":null}}}"""
+    val invalid_json = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-04-21T02:51:39ZZ\",\"params\":{\"resmsgid\":\"4544fce4-efee-4ee2-8816-fdb3f60ac492\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"status\":\"No dialcodeFound\"}}"
+    try
+      server.start(3000)
+    catch {
+      case e: IOException =>
+        System.out.println("Exception" + e)
+    }
+    server.enqueue(new MockResponse().setBody(user1))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-1")
+    server.enqueue(new MockResponse().setBody(clientError))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-9")
+    server.enqueue(new MockResponse().setBody(user2))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-2")
+    server.enqueue(new MockResponse().setBody(user3))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-3")
+    server.enqueue(new MockResponse().setBody(user4))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-4")
+    server.enqueue(new MockResponse().setBody(user2))
+    server.url("http://127.0.0.1:3000/learner/private/user/v1/read/user-5")
+    server.enqueue(new MockResponse().setHeader("Authorization", "auth_token"))
+
+  }
+
+  "UserCacheUpdater" should "be able to add user to cache with all information" in {
+    setupRestUtilData()
+    when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource)
+
+    val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil)
+    task.process()
+
+    /**
+      * Metrics Assertions
+      */
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.userCacheHit}").getValue() should be(7)
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.totalEventsCount}").getValue() should be(9)
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.skipCount}").getValue() should be(3)
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.successCount}").getValue() should be(7)
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.apiReadSuccessCount}").getValue() should be(7)
+    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.apiReadMissCount}").getValue() should be(0)
+
+    // select index: 12
+    jedis.select(userCacheConfig.userStore)
+
+    //user information: user-1
+    var userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-1")
+    userInfo.get("firstname") should be ("Utkarsha")
+    userInfo.get("lastname") should be ("Kapoor")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be ("BLOCK1")
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.get("schooludisecode") should be ("3183211")
+    userInfo.get("schoolname") should be ("DPS, MATHURA")
+    userInfo.get("usertype") should be ("administrator,teacher,other,parent")
+    userInfo.get("usersubtype") should be ("hm,crp")
+    userInfo.get("board") should be ("IGOT-Health")
+    userInfo.get("rootorgid") should be ("0127738024883077121")
+    userInfo.get("orgname") should be ("Root Org2")
+    userInfo.get("subject") should be ("""["IRCS"]""")
+    userInfo.get("language") should be ("""["English"]""")
+    userInfo.get("grade") should be ("""["Volunteers"]""")
+    userInfo.get("framework") should be ("igot_health")
+    userInfo.get("medium") should be ("""["English"]""")
+    userInfo.get("profileusertypes") should be ("""\[{"subType":"hm","type":"administrator"},{"type":"teacher"},{"subType":"crp","type":"administrator"},{"type":"other"},{"type":"parent"}\]""")
+
+    //user information: user-2
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-2")
+    userInfo.get("usersignintype") should be ("Self-Signed-In")
+    userInfo.keySet() should not contain theSameElementsAs(Seq("firstname", "lastname", "schoolname", "schooludisecode", "schoolname"))
+
+    /***
+      * user information: user-3
+      * Framework and block information is stored as empty string
+      */
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-3")
+    userInfo.get("firstname") should be ("Isha")
+    userInfo.get("lastname") should be ("Wakankar")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be (null)
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.get("schooludisecode") should be ("3183211")
+    userInfo.get("schoolname") should be ("DPS, MATHURA")
+    userInfo.get("usertype") should be ("administrator")
+    userInfo.get("usersubtype") should be ("deo")
+    userInfo.get("rootorgid") should be ("01285019302823526477")
+    userInfo.get("orgname") should be ("Root Org2")
+    userInfo.get("board") should be (null)
+    userInfo.get("subject") should be (null)
+    userInfo.get("language") should be ("""[]""")
+    userInfo.get("grade") should be (null)
+    userInfo.get("framework") should be (null)
+    userInfo.get("medium") should be (null)
+    userInfo.get("profileusertypes") should be ("""\[{"subType":"deo","type":"administrator"}\]""")
+
+    /***
+      * userinformation: user-4
+      * school information not present and subject not present in framework
+      */
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-4")
+    userInfo.get("firstname") should be ("Manju")
+    userInfo.get("lastname") should be ("Davanam")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be ("BLOCK1")
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.get("schooludisecode") should be (null)
+    userInfo.get("schoolname") should be (null)
+    userInfo.get("usertype") should be ("teacher")
+    userInfo.get("usersubtype") should be ("")
+    userInfo.get("board") should be ("")
+    userInfo.get("rootorgid") should be ("0127738024883077121")
+    userInfo.get("orgname") should be ("Root Org2")
+    userInfo.get("subject") should be ("""[]""")
+    userInfo.get("language") should be ("")
+    userInfo.get("grade") should be ("""["Volunteers"]""")
+    userInfo.get("framework") should be ("")
+    userInfo.get("medium") should be ("""["English"]""")
+    userInfo.get("phone") should be ("LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==")
+    userInfo.get("email") should be ("LednrgEat6NcG8DX3ue89T7osrjR76AWYqtSYzAvg1jvx1a4lZn6KssNuPP4UeiGoVPQ24MeJbTing10uKJND1TrTfg+K3pGBxLTV+B2SKBxLdBdWzwFOkCsEv53x7bP4T6a+wzaAmCWueMEdPmZuRg==")
+    userInfo.get("profileusertypes") should be ("""\[{"type":"teacher"}\]""")
+
+    /**
+      * UserId = user-5
+      * EData state is "Created"
+      * User SignupType is "sso"
+      * It should able to insert The Map(usersignintype, Validated)
+      *
+      */
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-5")
+    userInfo.get("usersignintype")
+
+    //user information: user-10
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-10")
+    userInfo.get("firstname") should be ("Utkarsha")
+    userInfo.get("lastname") should be ("Kapoor")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be ("""BLOCK1\]""")
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.get("schooludisecode") should be ("3183211")
+    userInfo.get("schoolname") should be ("""\[RPMMAT M.S UDHADIH""")
+    userInfo.get("usertype") should be ("administrator")
+    userInfo.get("usersubtype") should be ("")
+    userInfo.get("board") should be ("IGOT-Health")
+    userInfo.get("rootorgid") should be ("0127738024883077121")
+    userInfo.get("orgname") should be ("Root Org2")
+    userInfo.get("subject") should be ("""["IRCS"]""")
+    userInfo.get("language") should be ("""["English"]""")
+    userInfo.get("grade") should be ("""["Volunteers"]""")
+    userInfo.get("framework") should be ("igot_health")
+    userInfo.get("medium") should be ("""["English"]""")
+    userInfo.get("profileusertypes") should be ("""\[{"type":"administrator"}\]""")
+  }
+
+  "UserCacheUpdater" should "be able to add and update user record with different producer ids" in {
+    setupRestUtilData()
+    when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new LeanerInputSource)
+
+    val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil)
+    task.process()
+
+    // select index: 12
+    jedis.select(userCacheConfig.userStore)
+
+    //user information: user-1
+    var userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-1")
+    userInfo.get("firstname") should be ("Utkarsha")
+    userInfo.get("lastname") should be ("Kapoor")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be ("BLOCK1")
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.getOrDefault(userCacheConfig.userLoginTypeKey, null) should be (null)
+
+    when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new AppInputSource)
+    task.process()
+
+    userInfo = jedis.hgetAll(userCacheConfig.userStoreKeyPrefix +  "user-1")
+    userInfo.get("firstname") should be ("Utkarsha")
+    userInfo.get("lastname") should be ("Kapoor")
+    userInfo.get("state") should be ("Odisha")
+    userInfo.get("district") should be ("CUTTACK")
+    userInfo.get("block") should be ("BLOCK1")
+    userInfo.get("cluster") should be ("CLUSTER1")
+    userInfo.getOrDefault(userCacheConfig.userLoginTypeKey, null) should be ("teacher")
+  }
+
+  "UserCacheUpdater" should "throw exception" in intercept[Exception] {
+    setupRestUtilDataWithErrors
+    when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource)
+
+    val task = new UserCacheUpdaterStreamTaskV2(userCacheConfig, mockKafkaUtil)
+    task.process()
+
+  }
+}
+
+abstract class InputSourceBase[T] extends SourceFunction[T] {
+  override def cancel() = {}
+}
+
+class InputSource extends InputSourceBase[Event]{
+
+  override def run(ctx: SourceContext[Event]) {
+    val gson = new Gson()
+    EventFixture.telemetrEvents.foreach(f => {
+      val eventMap = gson.fromJson(f, new util.HashMap[String, Any]().getClass)
+      val event = new Event(eventMap)
+      event.kafkaKey()
+      ctx.collect(event)
+    })
+  }
+}
+
+class LeanerInputSource extends InputSourceBase[Event]{
+  override def run(ctx: SourceContext[Event]): Unit = {
+    val gson = new Gson()
+    val eventMap = gson.fromJson(EventFixture.telemetryEventWithLearnerPid, new util.HashMap[String, Any]().getClass)
+    val event = new Event(eventMap)
+    event.kafkaKey()
+    ctx.collect(event)
+  }
+}
+
+class AppInputSource extends InputSourceBase[Event]{
+  override def run(ctx: SourceContext[Event]): Unit = {
+    val gson = new Gson()
+    val eventMap = gson.fromJson(EventFixture.telemetryEventWithAppPid, new util.HashMap[String, Any]().getClass)
+    val event = new Event(eventMap)
+    event.kafkaKey()
+    ctx.collect(event)
+  }
+}
+
diff --git a/data-pipeline-flink/user-cache-updater/pom.xml b/data-pipeline-flink/user-cache-updater/pom.xml
deleted file mode 100644
index 7d9d9efe43..0000000000
--- a/data-pipeline-flink/user-cache-updater/pom.xml
+++ /dev/null
@@ -1,225 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.sunbird.dp</groupId>
-        <artifactId>pipeline-jobs</artifactId>
-        <version>1.0</version>
-    </parent>
-
-    <groupId>org.sunbird.dp.jobs</groupId>
-    <artifactId>user-cache-updater</artifactId>
-    <version>1.0.0</version>
-    <packaging>jar</packaging>
-    <name>UserCacheUpdater</name>
-    <description>
-        Updating user details into readis cache
-    </description>
-
-    <properties>
-        <encoding>UTF-8</encoding>
-        <scoverage.plugin.version>1.4.0</scoverage.plugin.version>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-scala_${scala.version}</artifactId>
-            <version>${flink.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.sunbird.dp</groupId>
-            <artifactId>dp-core</artifactId>
-            <version>1.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.sunbird.dp</groupId>
-            <artifactId>dp-core</artifactId>
-            <version>1.0.0</version>
-            <type>test-jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-test-utils_2.12</artifactId>
-            <version>1.10.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-runtime_2.12</artifactId>
-            <version>1.10.0</version>
-            <scope>test</scope>
-            <classifier>tests</classifier>
-        </dependency>
-        <dependency>
-            <groupId>it.ozimov</groupId>
-            <artifactId>embedded-redis</artifactId>
-            <version>0.7.1</version>
-            <scope>test</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.google.guava</groupId>
-                    <artifactId>guava</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.cassandraunit</groupId>
-            <artifactId>cassandra-unit</artifactId>
-            <version>3.1.1.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.flink</groupId>
-            <artifactId>flink-streaming-java_2.12</artifactId>
-            <version>1.10.0</version>
-            <scope>test</scope>
-            <classifier>tests</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_2.12</artifactId>
-            <version>3.0.6</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>3.3.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <sourceDirectory>src/main/scala</sourceDirectory>
-        <testSourceDirectory>src/test/scala</testSourceDirectory>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
-                <executions>
-                    <!-- Run shade goal on package phase -->
-                    <execution>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <shadedArtifactAttached>false</shadedArtifactAttached>
-                            <artifactSet>
-                                <excludes>
-                                    <exclude>com.google.code.findbugs:jsr305</exclude>
-                                </excludes>
-                            </artifactSet>
-                            <filters>
-                                <filter>
-                                    <!-- Do not copy the signatures in the META-INF folder.
-                                    Otherwise, this might cause SecurityExceptions when using the JAR. -->
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>META-INF/*.SF</exclude>
-                                        <exclude>META-INF/*.DSA</exclude>
-                                        <exclude>META-INF/*.RSA</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                            <transformers>
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
-                                    <mainClass>org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTask</mainClass>
-                                </transformer>
-                                <!-- append default configs -->
-                                <transformer
-                                        implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
-                                    <resource>reference.conf</resource>
-                                </transformer>
-                            </transformers>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <groupId>net.alchim31.maven</groupId>
-                <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
-                <configuration>
-                    <source>1.8</source>
-                    <target>1.8</target>
-                    <checkMultipleScalaVersions>false</checkMultipleScalaVersions>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>scala-compile-first</id>
-                        <phase>process-resources</phase>
-                        <goals>
-                            <goal>add-source</goal>
-                            <goal>compile</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>scala-test-compile</id>
-                        <phase>process-test-resources</phase>
-                        <goals>
-                            <goal>testCompile</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.22.2</version>
-                <configuration>
-                    <skipTests>true</skipTests>
-                </configuration>
-            </plugin>
-
-            <plugin>
-                <groupId>org.scalatest</groupId>
-                <artifactId>scalatest-maven-plugin</artifactId>
-                <version>1.0</version>
-                <configuration>
-                    <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
-                    <junitxml>.</junitxml>
-                    <filereports>usercache-testsuite.txt</filereports>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>test</id>
-                        <goals>
-                            <goal>test</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.scoverage</groupId>
-                <artifactId>scoverage-maven-plugin</artifactId>
-                <version>${scoverage.plugin.version}</version>
-                <configuration>
-                    <scalaVersion>${scala.version}</scalaVersion>
-                    <aggregate>true</aggregate>
-                    <highlighting>true</highlighting>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties b/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties
deleted file mode 100644
index 1be669fa75..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/resources/log4j.properties
+++ /dev/null
@@ -1,11 +0,0 @@
-# log4j.appender.file=org.apache.log4j.FileAppender
-log4j.appender.file=org.apache.log4j.RollingFileAppender
-log4j.appender.file.file=user-cache-updater.log
-log4j.appender.file.append=true
-log4j.appender.file.layout=org.apache.log4j.PatternLayout
-log4j.appender.file.MaxFileSize=256KB
-log4j.appender.file.MaxBackupIndex=4
-log4j.appender.file.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
-
-# Suppress the irrelevant (wrong) warnings from the Netty channel handler
-log4j.logger.org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline=ERROR, file
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf b/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf
deleted file mode 100644
index 5e34574265..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/resources/user-cache-updater.conf
+++ /dev/null
@@ -1,30 +0,0 @@
-include "base-config.conf"
-
-kafka {
-  input.topic = ${job.env}".telemetry.audit"
-  groupId = ${job.env}"-user-cache-updater-group"
-}
-
-task {
-  usercache.updater.parallelism = 1
-}
-
-# redis-metadata
-redis-meta {
-  database {
-    userstore.id = 4
-  }
-}
-
-lms-cassandra {
-  keyspace = "sunbird"
-  table {
-    user = "user"
-    location = "location"
-  }
-}
-
-user.self.signin.types = ["google","self"]
-user.validated.types = ["sso"]
-user.self.signin.key = "Self-Signed-In"
-user.valid.key = "Validated"
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala
deleted file mode 100644
index db9ac18f96..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/domain/Event.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.sunbird.dp.usercache.domain
-
-import java.util
-
-import org.sunbird.dp.core.domain.Events
-
-class Event(eventMap: util.Map[String, Any]) extends Events(eventMap) {
-
-  override def kafkaKey(): String = {
-    did()
-  }
-
-  def getId: String = {
-    Option(objectType()).map({ t => if (t.equalsIgnoreCase("User")) objectID() else null
-    }).getOrElse(null)
-  }
-
-  def getState: String = {
-    telemetry.read[String]("edata.state").getOrElse(null)
-  }
-
-  def getContextDataId(cDataType: String): String = {
-    val cdata = telemetry.read[util.ArrayList[util.Map[String, AnyRef]]]("context.cdata").getOrElse(null)
-    var signInType: String = null
-    Option(cdata).map(data => {
-      data.forEach(cdataMap => {
-        if (cdataMap.get("type").asInstanceOf[String].equalsIgnoreCase(cDataType)) signInType = cdataMap.get("id").toString else signInType
-      })
-    }).getOrElse(signInType)
-    signInType
-  }
-
-  def userMetaData(): util.ArrayList[String] = {
-    telemetry.read[util.ArrayList[String]]("edata.props").getOrElse(new util.ArrayList[String]())
-  }
-
-}
diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala
deleted file mode 100644
index 2a356cadec..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/functions/UserCacheUpdaterFunction.scala
+++ /dev/null
@@ -1,161 +0,0 @@
-package org.sunbird.dp.usercache.functions
-
-import java.util
-
-import com.datastax.driver.core.Row
-import com.datastax.driver.core.querybuilder.{Clause, QueryBuilder}
-import com.google.gson.Gson
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.streaming.api.functions.ProcessFunction
-import org.slf4j.LoggerFactory
-import org.sunbird.dp.core.cache.{DataCache, RedisConnect}
-import org.sunbird.dp.core.job.{BaseProcessFunction, Metrics}
-import org.sunbird.dp.core.util.CassandraUtil
-import org.sunbird.dp.usercache.domain.Event
-import org.sunbird.dp.usercache.task.UserCacheUpdaterConfig
-
-import scala.collection.JavaConverters._
-import scala.collection.mutable
-
-class UserCacheUpdaterFunction(config: UserCacheUpdaterConfig)(implicit val mapTypeInfo: TypeInformation[Event])
-  extends BaseProcessFunction[Event, Event](config) {
-
-  private[this] val logger = LoggerFactory.getLogger(classOf[UserCacheUpdaterFunction])
-  private var dataCache: DataCache = _
-  private var cassandraConnect: CassandraUtil = _
-
-  override def metricsList(): List[String] = {
-    List(config.dbReadSuccessCount, config.dbReadMissCount, config.userCacheHit, config.skipCount, config.successCount, config.totalEventsCount)
-  }
-
-  override def open(parameters: Configuration): Unit = {
-    super.open(parameters)
-    dataCache = new DataCache(config, new RedisConnect(config.metaRedisHost, config.metaRedisPort, config), config.userStore, config.userFields)
-    dataCache.init()
-    cassandraConnect = new CassandraUtil(config.cassandraHost, config.cassandraPort)
-  }
-
-  override def close(): Unit = {
-    super.close()
-    dataCache.close()
-  }
-
-  override def processElement(event: Event, context: ProcessFunction[Event, Event]#Context, metrics: Metrics): Unit = {
-    metrics.incCounter(config.totalEventsCount)
-    Option(event.getId).map(id => {
-      Option(event.getState).map(name => {
-        val userData: mutable.Map[String, AnyRef] = name.toUpperCase match {
-          case "CREATE" | "CREATED" => createAction(id, event, metrics)
-          case "UPDATE" | "UPDATED" => updateAction(id, event, metrics)
-          case _ => {
-            logger.info(s"Invalid event state name either it should be(Create/Created/Update/Updated) but found $name for ${event.mid()}")
-            metrics.incCounter(config.skipCount)
-            mutable.Map[String, AnyRef]()
-          }
-        }
-        if (!userData.isEmpty) {
-          dataCache.setWithRetry(id, new Gson().toJson(mapAsJavaMap(userData)))
-          metrics.incCounter(config.successCount)
-          metrics.incCounter(config.userCacheHit)
-        } else {
-          metrics.incCounter(config.skipCount)
-        }
-      }).getOrElse(metrics.incCounter(config.skipCount))
-    }).getOrElse(metrics.incCounter(config.skipCount))
-  }
-
-
-  /**
-   * When Edata.State is Create/Created then insert the user data into redis in string formate
-   */
-  def createAction(userId: String, event: Event, metrics: Metrics): mutable.Map[String, AnyRef] = {
-    val userData: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]()
-    Option(event.getContextDataId(cDataType = "SignupType")).map(signInType => {
-      if (config.userSelfSignedInTypeList.contains(signInType)) {
-        userData.put(config.userSignInTypeKey, config.userSelfSignedKey)
-      }
-      if (config.userValidatedTypeList.contains(signInType)) {
-        userData.put(config.userSignInTypeKey, config.userValidatedKey)
-      }
-    }).orNull
-    userData
-  }
-
-  /**
-   * When edata.state is update/updated then update the user metadta
-   * information by reading from the cassandra
-   */
-
-  def updateAction(userId: String, event: Event, metrics: Metrics): mutable.Map[String, AnyRef] = {
-    val userCacheData: mutable.Map[String, AnyRef] = dataCache.getWithRetry(userId)
-
-    Option(event.getContextDataId(cDataType = "UserRole")).map(loginType => {
-      userCacheData.put(config.userLoginTypeKey, loginType)
-    })
-    val userMetaDataList = event.userMetaData()
-    if (!userMetaDataList.isEmpty && userCacheData.contains(config.userSignInTypeKey) && ("Anonymous" != userCacheData.get(config.userSignInTypeKey).toString)) {
-
-      // Get the user details from the cassandra table
-      val userDetails: mutable.Map[String, AnyRef] = userCacheData.++(extractUserMetaData(readFromCassandra(
-        keyspace = config.keySpace,
-        table = config.userTable,
-        QueryBuilder.eq("id", userId),
-        metrics)
-      ))
-
-      // Fetching the location details from the cassandra table
-      val updatedUserDetails: mutable.Map[String, AnyRef] = userDetails.++(extractLocationMetaData(readFromCassandra(
-        keyspace = config.keySpace,
-        table = config.locationTable,
-        clause = QueryBuilder.in("id", userDetails.getOrElse("locationids", new util.ArrayList()).asInstanceOf[util.ArrayList[String]]),
-        metrics)
-      ))
-      logger.info(s"User details ( $userId ) are fetched from the db's and updating the redis now.")
-      updatedUserDetails
-    } else {
-      logger.info(s"Skipping the event update from databases since event Does not have user properties or user sigin in type is Anonymous ")
-      userCacheData
-    }
-  }
-
-  def readFromCassandra(keyspace: String, table: String, clause: Clause, metrics: Metrics): util.List[Row] = {
-    var rowSet: util.List[Row] = null
-    val query = QueryBuilder.select.all.from(keyspace, table).where(clause).toString
-    rowSet = cassandraConnect.find(query)
-    if (null != rowSet && !rowSet.isEmpty) {
-      metrics.incCounter(config.dbReadSuccessCount)
-      rowSet
-    } else {
-      metrics.incCounter(config.dbReadMissCount)
-      new util.ArrayList[Row]()
-    }
-
-  }
-
-  def extractUserMetaData(userDetails: util.List[Row]): mutable.Map[String, AnyRef] = {
-    val result: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]()
-    if (null != userDetails && !userDetails.isEmpty) {
-      val row: Row = userDetails.get(0)
-      val columnDefinitions = row.getColumnDefinitions()
-      val columnCount = columnDefinitions.size
-      for (i <- 0 until columnCount) {
-        result.put(columnDefinitions.getName(i), row.getObject(i))
-      }
-    }
-    result
-  }
-
-  def extractLocationMetaData(locationDetails: util.List[Row]): mutable.Map[String, AnyRef] = {
-    val result: mutable.Map[String, AnyRef] = mutable.Map[String, AnyRef]()
-    locationDetails.forEach((record: Row) => {
-      record.getString("type").toLowerCase match {
-        case config.stateKey => result.put(config.stateKey, record.getString("name"))
-        case config.districtKey => result.put(config.districtKey, record.getString("name"))
-        case _ => // Do nothing
-      }
-    })
-    result
-  }
-
-}
diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala
deleted file mode 100644
index 2f0e71ef4b..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterConfig.scala
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.sunbird.dp.usercache.task
-
-import com.typesafe.config.Config
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.TypeExtractor
-import org.sunbird.dp.core.job.BaseJobConfig
-import org.sunbird.dp.usercache.domain.Event
-
-import java.util.{List => JList}
-
-class UserCacheUpdaterConfig(override val config: Config) extends BaseJobConfig(config, "UserCacheUpdaterJob") {
-
-  private val serialVersionUID = 2905979434303791379L
-
-  implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
-
-  // Kafka Topics Configuration
-  val inputTopic: String = config.getString("kafka.input.topic")
-  val userFields = List("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype","locationids")
-
-  // User cache updater job metrics
-  val userCacheHit = "user-cache-hit"
-  val skipCount = "skipped-message-count"
-  val successCount = "success-message-count"
-  val dbReadSuccessCount = "db-read-success-count"
-  val dbReadMissCount = "db-read-miss-count"
-  val totalEventsCount ="total-audit-events-count"
-
-  val userSelfSignedInTypeList: JList[String] = config.getStringList("user.self.signin.types")
-  val userValidatedTypeList: JList[String] = config.getStringList("user.validated.types")
-  val userSelfSignedKey: String = config.getString("user.self.signin.key")
-  val userValidatedKey: String = config.getString("user.valid.key")
-
-  // Redis
-  val userStore: Int = config.getInt("redis-meta.database.userstore.id")
-
-  // lms-cassandra
-  val keySpace: String = config.getString("lms-cassandra.keyspace")
-  val locationTable: String = config.getString("lms-cassandra.table.location")
-  val userTable: String = config.getString("lms-cassandra.table.user")
-  val cassandraHost: String =  config.getString("lms-cassandra.host")
-  val cassandraPort: Int =  config.getInt("lms-cassandra.port")
-
-  val userCacheParallelism: Int = config.getInt("task.usercache.updater.parallelism")
-
-  // constants
-  val userSignInTypeKey = "usersignintype"
-  val userLoginTypeKey = "userlogintype"
-  val stateKey = "state"
-  val districtKey = "district"
-
-  // Consumers
-  val userCacheConsumer = "user-cache-consumer"
-
-  // Functions
-  val userCacheUpdaterFunction = "UserCacheUpdaterFunction"
-
-}
diff --git a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala b/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala
deleted file mode 100644
index dcb0606948..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/main/scala/org/sunbird/dp/usercache/task/UserCacheUpdaterStreamTask.scala
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.sunbird.dp.usercache.task
-
-import java.io.File
-
-import com.typesafe.config.ConfigFactory
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.TypeExtractor
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment
-import org.sunbird.dp.core.job.FlinkKafkaConnector
-import org.sunbird.dp.core.util.FlinkUtil
-import org.sunbird.dp.usercache.domain.Event
-import org.sunbird.dp.usercache.functions.UserCacheUpdaterFunction
-
-class UserCacheUpdaterStreamTask(config: UserCacheUpdaterConfig, kafkaConnector: FlinkKafkaConnector) {
-
-  private val serialVersionUID = -7729362727131516112L
-
-  def process(): Unit = {
-
-    implicit val env: StreamExecutionEnvironment = FlinkUtil.getExecutionContext(config)
-    implicit val eventTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
-
-    val source = kafkaConnector.kafkaEventSource[Event](config.inputTopic)
-    env.addSource(source, config.userCacheConsumer).uid(config.userCacheConsumer).rebalance()
-      .process(new UserCacheUpdaterFunction(config)).setParallelism(config.userCacheParallelism)
-      .name(config.userCacheUpdaterFunction).uid(config.userCacheUpdaterFunction)
-    env.execute(config.jobName)
-  }
-
-}
-
-// $COVERAGE-OFF$ Disabling scoverage as the below code can only be invoked within flink cluster
-object UserCacheUpdaterStreamTask {
-
-  def main(args: Array[String]): Unit = {
-    val configFilePath = Option(ParameterTool.fromArgs(args).get("config.file.path"))
-    val config = configFilePath.map {
-      path => ConfigFactory.parseFile(new File(path)).resolve()
-    }.getOrElse(ConfigFactory.load("user-cache-updater.conf").withFallback(ConfigFactory.systemEnvironment()))
-    val userCacheUpdaterConfig = new UserCacheUpdaterConfig(config)
-    val kafkaUtil = new FlinkKafkaConnector(userCacheUpdaterConfig)
-    val task = new UserCacheUpdaterStreamTask(userCacheUpdaterConfig, kafkaUtil)
-    task.process()
-  }
-}
-
-// $COVERAGE-ON$
diff --git a/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql b/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql
deleted file mode 100644
index a98159edb6..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/test/resources/data.cql
+++ /dev/null
@@ -1,112 +0,0 @@
-CREATE KEYSPACE sunbird WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 3};
-
-CREATE TABLE sunbird.user (
-    id text PRIMARY KEY,
-    avatar text,
-    channel text,
-    countrycode text,
-    createdby text,
-    createddate text,
-    currentlogintime text,
-    dob text,
-    email text,
-    emailverified boolean,
-    firstname text,
-    flagsvalue int,
-    framework map<text, frozen<list<text>>>,
-    gender text,
-    grade list<text>,
-    isdeleted boolean,
-    language list<text>,
-    lastlogintime text,
-    lastname text,
-    location text,
-    locationids list<text>,
-    loginid text,
-    maskedemail text,
-    maskedphone text,
-    password text,
-    phone text,
-    phoneverified boolean,
-    prevusedemail text,
-    prevusedphone text,
-    profilesummary text,
-    profilevisibility map<text, text>,
-    provider text,
-    recoveryemail text,
-    recoveryphone text,
-    registryid text,
-    roles list<text>,
-    rootorgid text,
-    status int,
-    subject list<text>,
-    tcstatus text,
-    tcupdateddate text,
-    temppassword text,
-    thumbnail text,
-    tncacceptedon timestamp,
-    tncacceptedversion text,
-    updatedby text,
-    updateddate text,
-    userid text,
-    username text,
-    usertype text,
-    webpages list<frozen<map<text, text>>>
-) WITH bloom_filter_fp_chance = 0.01
-    AND caching = {'keys': 'ALL', 'rows_per_partition': 'NONE'}
-    AND comment = ''
-    AND compaction = {'class': 'org.apache.cassandra.db.compaction.SizeTieredCompactionStrategy', 'max_threshold': '32', 'min_threshold': '4'}
-    AND compression = {'chunk_length_in_kb': '64', 'class': 'org.apache.cassandra.io.compress.LZ4Compressor'}
-    AND crc_check_chance = 1.0
-    AND dclocal_read_repair_chance = 0.1
-    AND default_time_to_live = 0
-    AND gc_grace_seconds = 864000
-    AND max_index_interval = 2048
-    AND memtable_flush_period_in_ms = 0
-    AND min_index_interval = 128
-    AND read_repair_chance = 0.0
-    AND speculative_retry = '99PERCENTILE';
-CREATE INDEX inx_u_userid ON sunbird.user (userid);
-CREATE INDEX inx_u_email ON sunbird.user (email);
-CREATE INDEX inx_u_loginid ON sunbird.user (loginid);
-CREATE INDEX inx_u_status ON sunbird.user (status);
-CREATE INDEX inx_u_username ON sunbird.user (username);
-CREATE INDEX inx_u_phone ON sunbird.user (phone);
-
-INSERT INTO sunbird.user (
-    id,
-    avatar,
-    channel ,
-    countrycode,
-    createdby,
-    createddate,
-    currentlogintime,
-    dob,
-    email,
-    emailverified,
-    firstname,
-    flagsvalue,
-    gender,
-    lastname,
-    location,
-    maskedemail,
-    maskedphone,
-    password,
-    phone,
-    phoneverified
-)
-VALUES ('user-3', 'VOS','sunbird', 'IN','MANJU','24-apr-2020','24-apr-2020','24-08-1993','manjunathd@ilimi.in',true,'Manjunath', 1, 'Male','Davanam','Banglore','********gmail.com','******181', '*******','4682647342',true);
-
-CREATE TABLE sunbird.location (
-    id text PRIMARY KEY,
-    code text,
-    name text,
-    parentid text,
-    type text
-);
-
-CREATE INDEX inx_loc_code ON sunbird.location (code);
-
-INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-1','0f5f0702000f6', 'KARNATAKA', 'state');
-INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-2','g0507020f00f6', 'TUMKUR', 'district');
-INSERT INTO sunbird.location(id,code,name,type) VALUES ('location-3','331211','MANVI','block');
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf b/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf
deleted file mode 100644
index b66d4a4bd6..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/test/resources/test.conf
+++ /dev/null
@@ -1,29 +0,0 @@
-include "base-test.conf"
-
-kafka {
-  input.topic = "flink.telemetry.audit"
-  groupId = "flink-user-cache-updater-group"
-}
-
-task {
-  usercache.updater.parallelism = 1
-}
-
-redis-meta {
-  database {
-    userstore.id = 4
-  }
-}
-
-lms-cassandra {
-  keyspace = "sunbird"
-  table {
-      user = "user"
-      location = "location"
-  }
-}
-
-user.self.signin.types = ["google","self"]
-user.validated.types = ["sso"]
-user.self.signin.key = "Self-Signed-In"
-user.valid.key = "Validated"
\ No newline at end of file
diff --git a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala b/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
deleted file mode 100644
index 9d42ae92c2..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/fixture/EventFixture.scala
+++ /dev/null
@@ -1,125 +0,0 @@
-package org.sunbird.dp.fixture
-
-object EventFixture {
-
-  val userCacheData3 = """{"usersignintype":"Validated","usertype":"TEACHER"}"""
-  val userCacheData4 = """{"channel":"KV123","phoneverified":false,"createdby":"c8e51123-61a3-454d-beb0-2202450b0096","subject":["English"],"email":"BJAguqy3GaJECrYqDUPjeducVxa5J9ZsW9A8qc7YHelkV7KbgkCKW10quCbhpgxbh2t4toXC8uXW\\ngiguS+8ucwzbmgPm7q7YSYz26SfpHnzBo/0Vh3TWqr2MOq9LlX6gT6a+wzaAmCWueMEdPmZuRg==","username":"I+CyiN6Bx0GCRm9lkA3xn5uNBm0AODhxeDwJebxxBfuGJ5V2v1R8v1PEQsP+V+y9sAFcM2WtaMLj\\n91hpzBq0PFcQTq6OSPQOm0sySPXTDzyLvm1cKaLwzvJ6fzLLs9nKT6a+wzaAmCWueMEdPmZuRg==","firstname":"A512","framework":{},"userid":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","usertype":"TEACHER","rootorgid":"0126978705345576967","id":"610bab7d-1450-4e54-bf78-c7c9b14dbc81","language":[],"grade":[],"roles":["BOOK_REVIEWER"],"status":1,"webpages":[],"createddate":"2019-04-11 08:58:16:512+0000","emailverified":true,"isdeleted":false,"locationids":["location-1","location-2","location-3"],"maskedemail":"a5**@yopmail.com","profilevisibility":{},"loginid":"I+CyiN6Bx0GCRm9lkA3xnx2W8+QgN39Y0We3KjR98O8hD6YjyoCirIBDsWHGwRf65PY/Cx+pFFK1\\nIz1VinIaKgDnSQwkl7ajzQjjRTzQbKOyHsAXkJgo9I5l7ulEYVXRT6a+wzaAmCWueMEdPmZuRg==","usersignintype":"Self-Signed-In","userlogintype":"Student","state":"Telangana","district":"Hyderabad"}"""
-
-  val telemetrEvents: List[String] = List(
-
-    /**
-     * UserId = 89490534-126f-4f0b-82ac-3ff3e49f3468
-     * EData state is "Created"
-     * User SignupType is "sso"
-     * It should able to insert The Map(usersignintype, Validated)
-     *
-     */
-    """
-      |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-1"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin,
-
-    /**
-     * UserId = user-2
-     * EData state is "create"
-     * User SignupType is "google"
-     * It should able to insert The Map(usersignintype, Self-Signed-In)
-     */
-
-
-    """
-      |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"user-2"},{"type":"SignupType","id":"google"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"user-2"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}""".stripMargin,
-
-    /**
-     * User-Id :user-3
-     * Edata.state is update
-     *
-     */
-    """
-      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-3", "type":"user"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-      |
-      |""".stripMargin,
-
-    /**
-     * UseId-Id: user-4
-     * Location id's are defined in the events and also loaded into redis db
-     */
-
-    """
-      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-3","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"Updated","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy","locationIds"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-      |
-      |""".stripMargin,
-
-
-    /**
-     * User Id - user-5
-     * Props are not defined
-     */
-    """
-      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-5","type":"user"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-      |
-      |""".stripMargin,
-
-
-    /**
-     * AUDIT Event with object type as content. (not related to user)
-     * Props are not defined - Skip Count should incr (skipCount =+ 1)
-     */
-    """
-      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"do_r8r8rew97we9r8","type":"content"},"edata":{"state":"Updated"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-      |
-      |""".stripMargin,
-
-    /**
-     * AUDIT Event, But having invalid state(other than create/update)
-     *
-     */
-
-    """
-      |
-      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{"state":"wrongState"},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-      |""".stripMargin,
-
-
-  /**
-   * AUDIT Event, But having without state id null
-   *
-   */
-
-  """
-    |
-    |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"user-5","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"UserRole"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"user-4","type":"user"},"edata":{},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-    |""".stripMargin
-
-
-
-
-    //      |{"eid":"AUDIT","ets":1573121861118,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606dfdd6","actor":{"id":"627a431d-4f5c-4adc-812d-1f01c5588555","type":"User"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"627a431d-4f5c-4adc-812d-1f01c5588555","type":"User"},"edata":{"state":"Create","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1573121861125,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-    //      |""".stripMargin,
-    //
-    //    """
-    //      |{"eid":"AUDIT","ets":1.573121861118E12,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","cdata":[{"id":"25cb0530-7c52-ecb1-cff2-6a14faab7910","type":"SignupType"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"5609876543h2fd34h5678jf909876af54345678"},"edata":{"state":"Update","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1.573121861125E12,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-    //      |""".stripMargin,
-    //
-    //    """
-    //      |{"eid":"AUDIT","ets":1.573121861118E12,"ver":"3.0","mid":"1573121861118.40f9136b-1cc3-458d-a04a-4459606df","actor":{"id":"5609876543234567890987654345678","type":"Request"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.5.0"},"env":"User","did":"2bcfc645e27e64625f7bad6ce282f9d0","rollup":{"l1":"01285019302823526477"}},"object":{"type":"User","id":"5609876543h2fd34h5678jf909876af54345678"},"edata":{"state":"Create","props":["recoveryEmail","recoveryPhone","userId","id","externalIds","updatedDate","updatedBy"]},"syncts":1.573121861125E12,"@timestamp":"2019-11-07T10:17:41.125Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-11-07T10:17:41.118+0000"}
-    //      |""".stripMargin,
-    //
-    //    """
-    //      |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}
-    //      |""".stripMargin
-    //    """
-    //      |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Created","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}
-    //      |""".stripMargin,
-    //    """
-    //      |{"actor":{"type":"Consumer","id":"89490534-126f-4f0b-82ac-3ff3e49f3468"},"eid":"AUDIT","edata":{"state":"Create","props":["firstName","email","emailVerified","id","userId","createdBy","rootOrgId","channel","userType","roles","phoneVerified","isDeleted","createdDate","status","userName","loginId","externalIds"]},"ver":"3.0","ets":1561739226844,"context":{"channel":"0126684405014528002","pdata":{"pid":"learner-service","ver":"2.0.0","id":"prod.diksha.learning.service"},"env":"User","cdata":[{"type":"User","id":"34881c3a-8b92-4a3c-a982-7f946137cb09"},{"type":"SignupType","id":"sso"},{"type":"Source","id":"android"},{"type":"Request","id":"91f3c280-99c1-11e9-956e-6b6ef71ed575"}],"rollup":{"l1":"0126684405014528002"}},"mid":"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8","object":{"type":"User"},"syncts":1561739243532,"@timestamp":"2019-06-28T16:27:23.532Z","flags":{"tv_processed":true},"type":"events"}
-    //      |""".stripMargin,
-    //    """
-    //      |{"actor":{"type":"System","id":"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d"},"eid":"AUDIT","edata":{"state":"Update","props":["medium","board","grade","syllabus","gradeValue"]},"ver":"3.0","ets":1561739240727,"context":{"pdata":{"pid":"sunbird.app","ver":"2.1.92","id":"prod.diksha.app"},"channel":"505c7c48ac6dc1edc9b08f21db5a571d","env":"sdk","did":"010612971a80a7677d0a3e849ab35cb4a83157de","cdata":[{"type":"UserRole","id":"student"}],"sid":"ea68a05e-0843-4c06-9a84-9b98cd974724"},"mid":"0268860e-76b0-4b4e-b99b-ebf543e7a9d8","object":{"id":"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d","type":"User","version":"","rollup":{}},"syncts":1561739245463,"@timestamp":"2019-06-28T16:27:25.463Z","flags":{"tv_processed":true},"type":"events"}
-    //      |""".stripMargin,
-    //    """
-    //      |{"eid":"AUDIT","ets":1571297660511,"ver":"3.0","mid":"1571297660511.32f5024a-aa30-4c82-abd8-bb8d8914ed2d","actor":{"id":"ef70da5a-bb99-4785-b970-1d6d6ee75aad","type":"User"},"context":{"channel":"01285019302823526477","pdata":{"id":"dev.sunbird.portal","pid":"learner-service","ver":"2.4.0"},"env":"User","did":"bfaf115f65a2086b062735885f4d2f1a","cdata":[{"id":"1b1392bc-39d8-6e47-d7d6-5781a2f1481a","type":"Request"}],"rollup":{"l1":"01285019302823526477"}},"object":{"id":"52226956-61d8-4c1b-b115-c660111866d3","type":"User"},"edata":{"state":"Update","props":["firstName","userId","id","externalIds","locationIds","updatedDate","updatedBy"]},"syncts":1571297660521,"@timestamp":"2019-10-17T07:34:20.521Z","flags":{"tv_processed":true,"dd_processed":true},"type":"events","ts":"2019-10-17T07:34:20.511+0000"}
-    //      |""".stripMargin
-
-  )
-
-
-}
diff --git a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala b/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala
deleted file mode 100644
index 6745cf9e88..0000000000
--- a/data-pipeline-flink/user-cache-updater/src/test/scala/org/sunbird/dp/spec/UserCacheUpdatetStreamTaskSpec.scala
+++ /dev/null
@@ -1,175 +0,0 @@
-package org.sunbird.dp.spec
-
-import java.util
-
-import com.google.gson.Gson
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.typeutils.TypeExtractor
-import org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration
-import org.apache.flink.streaming.api.functions.source.SourceFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
-import org.apache.flink.test.util.MiniClusterWithClientResource
-import org.cassandraunit.CQLDataLoader
-import org.cassandraunit.dataset.cql.FileCQLDataSet
-import org.cassandraunit.utils.EmbeddedCassandraServerHelper
-import org.mockito.Mockito
-import org.mockito.Mockito._
-import org.sunbird.dp.core.cache.RedisConnect
-import org.sunbird.dp.core.job.FlinkKafkaConnector
-import org.sunbird.dp.core.util.CassandraUtil
-import org.sunbird.dp.fixture.EventFixture
-import org.sunbird.dp.usercache.domain.Event
-import org.sunbird.dp.usercache.task.{UserCacheUpdaterConfig, UserCacheUpdaterStreamTask}
-import org.sunbird.dp.{BaseMetricsReporter, BaseTestSpec}
-import redis.clients.jedis.Jedis
-import redis.embedded.RedisServer
-
-class UserCacheUpdatetStreamTaskSpec extends BaseTestSpec {
-
-  implicit val mapTypeInfo: TypeInformation[Event] = TypeExtractor.getForClass(classOf[Event])
-
-  val flinkCluster = new MiniClusterWithClientResource(new MiniClusterResourceConfiguration.Builder()
-    .setConfiguration(testConfiguration())
-    .setNumberSlotsPerTaskManager(1)
-    .setNumberTaskManagers(1)
-    .build)
-
-  var redisServer: RedisServer = _
-  val config: Config = ConfigFactory.load("test.conf")
-  val userCacheConfig: UserCacheUpdaterConfig = new UserCacheUpdaterConfig(config)
-  val mockKafkaUtil: FlinkKafkaConnector = mock[FlinkKafkaConnector](Mockito.withSettings().serializable())
-  val gson = new Gson()
-  var jedis: Jedis = _
-
-  override protected def beforeAll(): Unit = {
-    super.beforeAll()
-    println("******Starting the Embedded Cassandra*******")
-    EmbeddedCassandraServerHelper.startEmbeddedCassandra(80000L)
-    val cassandraUtil = new CassandraUtil(userCacheConfig.cassandraHost, userCacheConfig.cassandraPort)
-    val session = cassandraUtil.session
-    val dataLoader = new CQLDataLoader(session)
-    dataLoader.load(new FileCQLDataSet(getClass.getResource("/data.cql").getPath, true, true));
-    testCassandraUtil(cassandraUtil)
-    redisServer = new RedisServer(6340)
-    redisServer.start()
-    BaseMetricsReporter.gaugeMetrics.clear()
-    val redisConnect = new RedisConnect(userCacheConfig.metaRedisHost, userCacheConfig.metaRedisPort, userCacheConfig)
-
-    jedis = redisConnect.getConnection(userCacheConfig.userStore)
-    setupRedisTestData(jedis)
-    flinkCluster.before()
-  }
-
-  override protected def afterAll(): Unit = {
-    super.afterAll()
-    redisServer.stop()
-    flinkCluster.after()
-  }
-
-  def setupRedisTestData(jedis: Jedis) {
-
-
-    // Insert user test data
-    jedis.set("user-3", EventFixture.userCacheData3)
-    jedis.set("user-4", EventFixture.userCacheData4)
-    jedis.close()
-  }
-
-  "UserCache Updater pipeline" should "Should able to add user data into cache" in {
-
-    when(mockKafkaUtil.kafkaEventSource[Event](userCacheConfig.inputTopic)).thenReturn(new InputSource)
-
-    val task = new UserCacheUpdaterStreamTask(userCacheConfig, mockKafkaUtil)
-    task.process()
-
-    /**
-     * Metrics Assertions
-     */
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.userCacheHit}").getValue() should be(5)
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.totalEventsCount}").getValue() should be(8)
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.dbReadSuccessCount}").getValue() should be(2)
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.dbReadMissCount}").getValue() should be(2)
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.skipCount}").getValue() should be(4)
-    BaseMetricsReporter.gaugeMetrics(s"${userCacheConfig.jobName}.${userCacheConfig.successCount}").getValue() should be(5)
-
-    /**
-     * UserId = 89490534-126f-4f0b-82ac-3ff3e49f3468
-     * EData state is "Created"
-     * User SignupType is "sso"
-     * It should able to insert The Map(usersignintype, Validated)
-     */
-    jedis.select(userCacheConfig.userStore)
-
-    val ssoUser = jedis.get("user-1")
-    ssoUser should not be null
-    val ssoUserMap: util.Map[String, AnyRef] = gson.fromJson(ssoUser, new util.LinkedHashMap[String, AnyRef]().getClass)
-    ssoUserMap.get("usersignintype") should be("Validated")
-
-    /**
-     * UserId = user-2
-     * EData state is "Created"
-     * User SignupType is "google"
-     * It should able to insert The Map(usersignintype, Self-Signed-In)
-     */
-    val googleUser = jedis.get("user-2")
-    googleUser should not be null
-    val googleUserMap: util.Map[String, AnyRef] = gson.fromJson(googleUser, new util.LinkedHashMap[String, AnyRef]().getClass)
-    googleUserMap.get("usersignintype") should be("Self-Signed-In")
-
-
-    // When action is Update and location id's are empty
-
-    val userInfo = jedis.get("user-3")
-    val userInfoMap: util.Map[String, AnyRef] = gson.fromJson(userInfo, new util.LinkedHashMap[String, AnyRef]().getClass)
-    userInfoMap.get("createdby") should be("MANJU")
-    userInfoMap.get("location") should be("Banglore")
-    userInfoMap.get("maskedphone") should be("******181")
-
-    // When action is Updated and location ids are present
-    val locationInfo = jedis.get("user-4")
-    val locationInfoMap: util.Map[String, AnyRef] = gson.fromJson(locationInfo, new util.LinkedHashMap[String, AnyRef]().getClass)
-
-    // Initially, in the redis for the user-4 is loaded with location details
-    // State - Telangana & District - Hyderabad
-    // Now it should be bellow assertions
-    locationInfoMap.get("state") should be("KARNATAKA")
-    locationInfoMap.get("district") should be("TUMKUR")
-
-
-    val emptyProps = jedis.get("user-5")
-    val emptyPropsMap: util.Map[String, AnyRef] = gson.fromJson(emptyProps, new util.LinkedHashMap[String, AnyRef]().getClass)
-    emptyPropsMap.get(userCacheConfig.userLoginTypeKey) should be("25cb0530-7c52-ecb1-cff2-6a14faab7910")
-
-  }
-
-  def testCassandraUtil(cassandraUtil: CassandraUtil): Unit = {
-    cassandraUtil.reconnect()
-    val response = cassandraUtil.findOne("SELECT * FROM sunbird.location;")
-    response should not be null
-    val upsert = cassandraUtil.upsert("SELECT * FROM sunbird.location;")
-    upsert should be(true)
-    cassandraUtil.getUDTType("sunbird", "test") should not be not
-    cassandraUtil.close()
-    val onSessionClose = cassandraUtil.find("SELECT * FROM sunbird.location;")
-    onSessionClose should not be null
-
-  }
-
-}
-
-class InputSource extends SourceFunction[Event] {
-
-  override def run(ctx: SourceContext[Event]) {
-    val gson = new Gson()
-    EventFixture.telemetrEvents.foreach(f => {
-      val eventMap = gson.fromJson(f, new util.HashMap[String, Any]().getClass)
-      val event = new Event(eventMap)
-      event.kafkaKey()
-      ctx.collect(event)
-    })
-  }
-
-  override def cancel() = {}
-}
-
diff --git a/data-pipeline/.gitignore b/data-pipeline/.gitignore
deleted file mode 100644
index 0435c14b33..0000000000
--- a/data-pipeline/.gitignore
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*.class
-*.war
-*.ear
-target/
-.classpath
-.project
-.vagrant
-.settings/
-.idea/
-.idea_modules/
-*.iml
-*.ipr
-*.iws
-*/.cache
-deploy
-*.swp
diff --git a/data-pipeline/README.md b/data-pipeline/README.md
deleted file mode 100644
index e0cd6c88b7..0000000000
--- a/data-pipeline/README.md
+++ /dev/null
@@ -1,26 +0,0 @@
-To create a working kafka-samza pipeline, do the following:
-- bin/grid bootstrap
-- mvn clean package
-- mkdir -p deploy/samza
-- tar -xvf ./target/ekstep-samza-0.0.1-dist.tar.gz -C deploy/samza
-
-Assuming that kafka is running
-- deploy/kafka/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic telemetry_events
-
-Starting a producer
-- deploy/kafka/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic telemetry_events
-
-Starting a consumer (to see if everything is working)
-- deploy/kafka/bin/kafka-console-consumer.sh  --zookeeper localhost:2181 --topic events_with_location
-
-Note: If job cannot get location of an event, the event will be posted to another topic called events_failed_location
-
-Sample events for producer
-- {"did":"bc811958-b4b7-4873-a43a-03718edba45b","edata":{"eks":{"loc":"12.9310593,77.6238299","ueksid":"sharan"}},"eid":"GE_SESSION_START","gdata":{"id":"genie.android","ver":"2.2.18"},"sid":"6d5d6eeb-4f1b-4eed-8641-ec9e1884a218","ts":"2015-07-14T12:43:47+05:30","uid":"31e1cbf2b23a01ea035ee3323fe2ab95950c8284","ver":"1.0"}
-- {"did":"bc811958-b4b7-4873-a43a-03718edba45b","edata":{"eks":{"err":"","gid":"org.ekstep.math.pp","length":9,"tmsize":0}},"eid":"GE_GAME_END","gdata":{"id":"genie.android","ver":"2.2.18"},"sid":"6d5d6eeb-4f1b-4eed-8641-ec9e1884a218","ts":"2015-07-14T12:43:59+05:30","uid":"31e1cbf2b23a01ea035ee3323fe2ab95950c8284","ver":"1.0"}
-
-Deploying job on YARN
-- deploy/samza/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/reverseSearch.properties
-
-So, simple setup will do the following
-test11 -> reverse_search -> test13
diff --git a/data-pipeline/assessment-aggregator/pom.xml b/data-pipeline/assessment-aggregator/pom.xml
deleted file mode 100644
index a0ee8418a8..0000000000
--- a/data-pipeline/assessment-aggregator/pom.xml
+++ /dev/null
@@ -1,164 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>assessment-aggregator</artifactId>
-    <version>0.0.4</version>
-    <name>AssessmentAggregator</name>
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.cassandraunit</groupId>
-            <artifactId>cassandra-unit</artifactId>
-            <version>3.1.1.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
-            <version>RELEASE</version>
-            <scope>compile</scope>
-        </dependency>
-    </dependencies>
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>
\ No newline at end of file
diff --git a/data-pipeline/assessment-aggregator/src/main/assembly/src.xml b/data-pipeline/assessment-aggregator/src/main/assembly/src.xml
deleted file mode 100644
index 70b36631b8..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/assessment-aggregator.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:assessment-aggregator</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties b/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties
deleted file mode 100644
index facaca80f6..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/config/assessment-aggregator.properties
+++ /dev/null
@@ -1,79 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-####Environment
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.AssessmentAggregator
-
-# YARN
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__assessment_aggregator_yarn_container_count__
-
-
-# Task
-task.class=org.ekstep.ep.samza.task.AssessmentAggregatorTask
-task.inputs=kafka.__env__.telemetry.assess
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot,jmx
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-metrics.reporter.jmx.class=org.apache.samza.metrics.reporter.JmxReporterFactory
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.msg.serde=string
-systems.kafka.samza.key.serde=string
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__
-systems.kafka.consumer.fetch.message.max.bytes=__assessment_aggregator_consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__assessment_aggregator_consumer_fetch_max_bytes__
-systems.kafka.samza.fetch.threshold=__assessment_aggregator_messages_fetch_threshold__
-
-
-## Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.failed.topic.name=__env__.telemetry.assess.failed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=db-hit-count,db-insert-count,db-update-count
-
-
-# sunbird middleware cassandra config
-middleware.cassandra.host=__middleware_cassandra_host__
-middleware.cassandra.port=__middleware_cassandra_port__
-middleware.cassandra.courses_keyspace=__middleware_cassandra__courses_keyspace__
-middleware.cassandra.aggregator_table=__middleware_cassandra_assessment_aggregator_table__
-middleware.cassandra.question_type=__middleware_cassandra_assessment_question_type__
\ No newline at end of file
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java
deleted file mode 100644
index ebd808cc5c..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Aggregate.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.datastax.driver.core.UDTValue;
-
-import java.text.DecimalFormat;
-import java.util.List;
-
-
-public class Aggregate {
-
-    private double totalScore;
-    private double totalMaxScore;
-    private List<UDTValue> questionsList;
-    private DecimalFormat df = new DecimalFormat("0.0#");
-
-    public Aggregate(double totalScore, double totalMaxScore, List<UDTValue> questionsList) {
-
-        this.totalScore = totalScore;
-        this.totalMaxScore = totalMaxScore;
-        this.questionsList = questionsList;
-    }
-
-    public double getTotalScore() {
-        return totalScore;
-    }
-
-
-    public double getTotalMaxScore() {
-        return totalMaxScore;
-    }
-
-    public String getGrandTotal() {
-        return String.format("%s/%s", df.format(totalScore), df.format(totalMaxScore));
-    }
-
-    public List<UDTValue> getQuestionsList() {
-        return questionsList;
-    }
-
-
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java
deleted file mode 100644
index 5cbcec432c..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/BatchEvent.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.reader.Telemetry;
-
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class BatchEvent {
-    private final Telemetry telemetry;
-
-    public BatchEvent(Map<String, Object> map) {
-        this.telemetry = new Telemetry(map);
-
-    }
-
-    public String getJson() {
-        Gson gson = new Gson();
-        return gson.toJson(telemetry.getMap());
-    }
-
-    public Long assessmentEts() {
-        NullableValue<Object> ets = telemetry.read("assessmentTs");
-        if (ets.value().getClass().equals(Double.class)) {
-            return ((Double) ets.value()).longValue();
-        }
-        return ((Long) ets.value());
-    }
-
-
-    public String courseId() {
-        NullableValue<String> courseid = telemetry.read("courseId");
-        return courseid.value();
-    }
-
-    public String contentId() {
-        NullableValue<String> worksheetId = telemetry.read("contentId");
-        return worksheetId.value();
-    }
-
-    public String batchId() {
-        NullableValue<String> batchId = telemetry.read("batchId");
-        return batchId.value();
-    }
-
-    public String attemptId() {
-        NullableValue<String> attemptId = telemetry.read("attemptId");
-        return attemptId.value();
-    }
-
-    public String userId() {
-        NullableValue<String> userId = telemetry.read("userId");
-        return userId.value();
-    }
-
-    public List<Map<String, Object>> assessEvents() {
-        NullableValue<List<Map<String, Object>>> assessEvents = telemetry.read("events");
-        return assessEvents.value();
-    }
-
-    public void markSkipped(BatchEvent batchEvent) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("flags.batch_aggregate_assess_skipped", true);
-        telemetry.add("metadata.aggregate_assess", batchEvent.attemptId() +
-                " : skipping batch Event older than last batch assessment time");
-    }
-
-
-    @Override
-    public String toString() {
-        return "BatchEvent{" +
-                "telemetry=" + telemetry +
-                '}';
-    }
-}
-
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java
deleted file mode 100644
index b9fb62dd68..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/Question.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-
-public class Question {
-    private String id;
-    private double maxscore;
-    private List<Map<String, Object>> params;
-    private String title;
-    private String type;
-    private String desc;
-
-    public Question(String id, double maxscore, List<Map<String, Object>> params, String title, String type, String desc) {
-        this.id = id;
-        this.maxscore = maxscore;
-        this.params = params;
-        this.title = title;
-        this.type = type;
-        this.desc = desc;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public double getMaxScore() {
-        return maxscore;
-    }
-
-
-    public List<Map<String, Object>> getParams() {
-        ListIterator<Map<String, Object>>
-                iterator = params.listIterator();
-        while (iterator.hasNext()) {
-            Map<String, Object> values = iterator.next();
-            for (Map.Entry<String, Object> entry : values.entrySet()) {
-                if (null!= entry.getValue() && (!(entry.getValue() instanceof String)))
-                    entry.setValue(new Gson().toJson(entry.getValue()));
-            }
-        }
-        return params;
-    }
-
-    public String getTitle() {
-        return title;
-    }
-
-
-    public String getType() {
-        return type;
-    }
-
-
-    public String getDesc() {
-        return desc;
-    }
-
-
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java
deleted file mode 100644
index 99e679b707..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/domain/QuestionData.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Map;
-
-public class QuestionData {
-
-
-    private List<Map<String, Object>> resvalues;
-    private double duration;
-    private double score;
-    private Question item;
-    private long assessts;
-
-    public QuestionData(List<Map<String, Object>> resvalues, double duration, double score, Question item) {
-
-        this.resvalues = resvalues;
-        this.duration = duration;
-        this.score = score;
-        this.item = item;
-    }
-
-
-    public List<Map<String, Object>> getResvalues() {
-        ListIterator<Map<String, Object>>
-                iterator = resvalues.listIterator();
-        while (iterator.hasNext()) {
-            Map<String, Object> values = iterator.next();
-            for (Map.Entry<String, Object> entry : values.entrySet()) {
-                if (null!= entry.getValue() && (!(entry.getValue() instanceof String)))
-                    entry.setValue(new Gson().toJson(entry.getValue()));
-            }
-        }
-        return resvalues;
-    }
-
-    public double getDuration() {
-        return duration;
-    }
-
-    public double getScore() {
-        return score;
-    }
-
-    public Question getItem() {
-        return item;
-    }
-
-    public long getEts() {
-        return assessts;
-    }
-
-    public void setEts(long ets) {
-        this.assessts = ets;
-    }
-
-}
-
-
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java
deleted file mode 100644
index 9fa13140a6..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/service/AssessmentAggregatorService.java
+++ /dev/null
@@ -1,97 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.UDTValue;
-import com.datastax.driver.core.exceptions.DriverException;
-import com.google.gson.Gson;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Aggregate;
-import org.ekstep.ep.samza.domain.BatchEvent;
-import org.ekstep.ep.samza.domain.QuestionData;
-import org.ekstep.ep.samza.task.AssessmentAggregatorConfig;
-import org.ekstep.ep.samza.task.AssessmentAggregatorSink;
-import org.ekstep.ep.samza.task.AssessmentAggregatorSource;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.ekstep.ep.samza.util.DBUtil;
-import org.joda.time.DateTime;
-
-import java.util.*;
-
-public class AssessmentAggregatorService {
-
-    private static Logger LOGGER = new Logger(AssessmentAggregatorService.class);
-    private DBUtil dbUtil;
-    private Comparator<QuestionData> byEts = (QuestionData o1, QuestionData o2) -> Long.compare(o2.getEts(), o1.getEts());
-
-    public AssessmentAggregatorService(CassandraConnect cassandraConnect, AssessmentAggregatorConfig config) {
-        this.dbUtil = new DBUtil(cassandraConnect, config);
-    }
-
-    public void process(AssessmentAggregatorSource source, AssessmentAggregatorSink sink) throws Exception {
-        try {
-            BatchEvent batchEvent = source.getEvent();
-            Row assessment = dbUtil.getAssessmentFromDB(batchEvent);
-            if (null != assessment) {
-            	sink.incDBHits();
-            }
-            if (null != assessment) {
-                Long last_attempted_on = assessment.getTimestamp("last_attempted_on").getTime();
-                if (batchEvent.assessmentEts() > last_attempted_on) {
-                	saveAssessment(assessment, batchEvent);
-                	sink.incDBUpdateCount();
-                	sink.batchSuccess();
-                } else {
-                	LOGGER.info(batchEvent.attemptId(), ": Batch Event older than last assessment time, skipping");
-                    sink.skip(batchEvent);
-                }
-            } else {
-            	saveAssessment(assessment, batchEvent);
-            	sink.incDBInsertCount();
-            	sink.batchSuccess();
-            }
-            
-        } catch (DriverException ex) {
-        	ex.printStackTrace();
-            LOGGER.error("", "Exception while fetching from db : " + ex);
-            throw new DriverException(ex);
-        } catch (Exception ex) {
-        	ex.printStackTrace();
-            LOGGER.error("", "Failed to parse the batchEvent: ", ex);
-            sink.fail(source.getMessage().toString());
-        }
-    }
-    
-    public void saveAssessment(Row assessment, BatchEvent batchEvent) {
-    	Long createdOn = null != assessment ? assessment.getTimestamp("created_on").getTime() : new DateTime().getMillis();
-        Aggregate assess = getAggregateData(batchEvent, createdOn);
-        dbUtil.updateAssessmentToDB(batchEvent, assess, createdOn);
-        LOGGER.info("", " Successfully Aggregated the batch event - batchid: " + batchEvent.batchId()
-                + " ,userid: " + batchEvent.userId() + " ,couserid: " + batchEvent.courseId()
-                + " ,contentid: " + batchEvent.contentId());
-    }
-
-    public Aggregate getAggregateData(BatchEvent batchEvent, Long createdOn) {
-        double totalMaxScore = 0;
-        double totalScore = 0;
-        List<UDTValue> questionsList = new ArrayList<>();
-        TreeSet<QuestionData> questionSet = new TreeSet<QuestionData>(byEts);
-        HashMap<String, String> checkDuplicate = new HashMap<String, String>();
-        for (Map<String, Object> event : batchEvent.assessEvents()) {
-            if (event.containsKey("edata")) {
-                QuestionData questionData = new Gson().fromJson(new Gson().toJson(event.get("edata")), QuestionData.class);
-                questionData.setEts(((Number) event.get("ets")).longValue());
-                questionSet.add(questionData);
-            }
-        }
-        for (QuestionData questionData : questionSet) {
-            if (!checkDuplicate.containsKey(questionData.getItem().getId())) {
-                totalScore += questionData.getScore();
-                totalMaxScore += questionData.getItem().getMaxScore();
-                questionsList.add(dbUtil.getQuestion(questionData));
-                checkDuplicate.put(questionData.getItem().getId(), "");
-            }
-        }
-        return new Aggregate(totalScore, totalMaxScore, questionsList);
-    }
-    
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java
deleted file mode 100644
index b3157feac6..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorConfig.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-public class AssessmentAggregatorConfig {
-
-    private final String JOB_NAME = "AssessmentAggregator";
-    private String failedTopic;
-    private final String coursesKeyspace;
-    private final String assessementTable;
-    private final String assessmentQuestionUDT;
-    private final String cassandraHost;
-    private final int cassandraPort;
-
-    public AssessmentAggregatorConfig(Config config) {
-    	
-        coursesKeyspace = config.get("middleware.cassandra.courses_keyspace", "sunbird_courses");
-        assessementTable = config.get("middleware.cassandra.aggregator_table", "assessment_aggregator");
-        assessmentQuestionUDT = config.get("middleware.cassandra.question_type", "question");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.assess.failed");
-        cassandraHost = config.get("middleware.cassandra.host", "127.0.0.1");
-        cassandraPort = config.getInt("middleware.cassandra.port", 9042);
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String getCoursesKeyspace() {
-        return coursesKeyspace;
-    }
-
-    public String getAssessementTable() {
-        return assessementTable;
-    }
-
-    public String getAssessmentQuestionUDT() {
-        return assessmentQuestionUDT;
-    }
-
-    public String getCassandraHost() { return cassandraHost; }
-
-    public int getCassandraPort() { return  cassandraPort; }
-
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java
deleted file mode 100644
index 939e94a262..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSink.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.BatchEvent;
-
-public class AssessmentAggregatorSink extends BaseSink {
-
-    private AssessmentAggregatorConfig config;
-
-    public AssessmentAggregatorSink(MessageCollector collector, JobMetrics metrics,
-                                    AssessmentAggregatorConfig config) {
-        super(collector, metrics);
-        this.config = config;
-    }
-
-    public void batchSuccess() {
-        metrics.incBatchSuccessCounter();
-    }
-
-    public void success() {
-        metrics.incSuccessCounter();
-    }
-
-    public void skip(BatchEvent batchEvent) {
-        batchEvent.markSkipped(batchEvent);
-        toTopic(config.failedTopic(), batchEvent.attemptId(), batchEvent.getJson());
-        metrics.incSkippedCounter();
-    }
-
-    public void fail(String message) {
-        toTopic(config.failedTopic(), null, message);
-        metrics.incFailedCounter();
-    }
-
-    public void incDBHits() {
-        metrics.incDBHitCount();
-    }
-    
-    public void incDBInsertCount() {
-    	metrics.incDBInsertCount();
-    }
-    
-    public void incDBUpdateCount() {
-    	metrics.incDBUpdateCount();
-    }
-}
-
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java
deleted file mode 100644
index 8e0cf70671..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorSource.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.domain.BatchEvent;
-
-import java.util.Map;
-
-public class AssessmentAggregatorSource {
-    private IncomingMessageEnvelope envelope;
-
-    public AssessmentAggregatorSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Object getMessage()
-    {
-        return envelope.getMessage();
-    }
-
-    @SuppressWarnings("unchecked")
-    public BatchEvent getEvent() {
-        String message = (String) envelope.getMessage();
-        Map<String, Object> jsonMap = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-        return new BatchEvent(jsonMap);
-    }
-
-}
-
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java
deleted file mode 100644
index f7770d3be6..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/task/AssessmentAggregatorTask.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.*;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.AssessmentAggregatorService;
-import org.ekstep.ep.samza.util.CassandraConnect;
-
-public class AssessmentAggregatorTask extends BaseSamzaTask {
-
-    private JobMetrics metrics;
-    private AssessmentAggregatorService service;
-    private AssessmentAggregatorConfig config;
-
-    public AssessmentAggregatorTask(Config config, TaskContext taskContext, CassandraConnect cassandraConnect) {
-        init(config, taskContext, cassandraConnect);
-    }
-
-    public AssessmentAggregatorTask() {
-
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) {
-        init(config, context, null);
-    }
-
-    private void init(Config config, TaskContext context, CassandraConnect cassandraConnect) {
-
-        this.config = new AssessmentAggregatorConfig(config);
-        this.metrics = new JobMetrics(context, this.config.jobName());
-        cassandraConnect = null != cassandraConnect ? cassandraConnect : new CassandraConnect(this.config.getCassandraHost(), this.config.getCassandraPort());
-        this.service = new AssessmentAggregatorService(cassandraConnect, this.config);
-        this.initTask(config, this.metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-        AssessmentAggregatorSource source = new AssessmentAggregatorSource(envelope);
-        AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, metrics, config);
-        service.process(source, sink);
-    }
-
-    public JobMetrics getJobMetrics() {
-    	return this.metrics;
-    }
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java b/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java
deleted file mode 100644
index 49ff425ad3..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/java/org/ekstep/ep/samza/util/DBUtil.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.UDTValue;
-import com.datastax.driver.core.UserType;
-import com.datastax.driver.core.querybuilder.Insert;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import org.ekstep.ep.samza.domain.Aggregate;
-import org.ekstep.ep.samza.domain.BatchEvent;
-import org.ekstep.ep.samza.domain.QuestionData;
-import org.ekstep.ep.samza.task.AssessmentAggregatorConfig;
-import org.joda.time.DateTime;
-
-import java.math.BigDecimal;
-import java.sql.Timestamp;
-
-
-public class DBUtil {
-    private CassandraConnect cassandraConnect;
-    private AssessmentAggregatorConfig config;
-    private UserType questionType;
-
-    public DBUtil(CassandraConnect cassandraConnect, AssessmentAggregatorConfig config) {
-        this.cassandraConnect = cassandraConnect;
-        this.config = config;
-        questionType = cassandraConnect.getUDTType(config.getCoursesKeyspace(), config.getAssessmentQuestionUDT());
-    }
-
-    public UDTValue getQuestion(QuestionData questionData) {
-        return questionType.newValue().setString("id", questionData.getItem().getId())
-                .setDouble("max_score", questionData.getItem().getMaxScore())
-                .setDouble("score", questionData.getScore())
-                .setString("type", questionData.getItem().getType())
-                .setString("title", questionData.getItem().getTitle())
-                .setList("resvalues", questionData.getResvalues())
-                .setList("params", questionData.getItem().getParams())
-                .setString("description", questionData.getItem().getDesc())
-                .setDecimal("duration", BigDecimal.valueOf(questionData.getDuration()))
-                .setTimestamp("assess_ts", new Timestamp(questionData.getEts()));
-
-    }
-
-    public void updateAssessmentToDB(BatchEvent batchEvent, Aggregate aggregate, Long createdOn) {
-
-        Insert query = QueryBuilder.insertInto(config.getCoursesKeyspace(), config.getAssessementTable())
-                .value("course_id", batchEvent.courseId())
-                .value("batch_id", batchEvent.batchId()).value("user_id", batchEvent.userId())
-                .value("content_id", batchEvent.contentId()).value("attempt_id", batchEvent.attemptId())
-                .value("updated_on", new DateTime().getMillis()).value("created_on", createdOn)
-                .value("last_attempted_on", batchEvent.assessmentEts()).value("total_score", aggregate.getTotalScore())
-                .value("total_max_score", aggregate.getTotalMaxScore()).value("question", aggregate.getQuestionsList())
-                .value("grand_total", aggregate.getGrandTotal());
-
-        cassandraConnect.upsert(query);
-    }
-
-
-    public Row getAssessmentFromDB(BatchEvent event) {
-        String query = QueryBuilder.select("last_attempted_on", "created_on").from(config.getCoursesKeyspace(), config.getAssessementTable())
-                .where(QueryBuilder.eq("attempt_id", event.attemptId())).and(QueryBuilder.eq("batch_id", event.batchId()))
-                .and(QueryBuilder.eq("user_id", event.userId())).and(QueryBuilder.eq("content_id", event.contentId()))
-                .and(QueryBuilder.eq("course_id", event.courseId())).toString();
-        return cassandraConnect.findOne(query);
-    }
-}
diff --git a/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml b/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/assessment-aggregator/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index 25d08ac64f..0000000000
--- a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,1525 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-
-import java.util.Map;
-
-public class EventFixture {
-
-    public static final String BATCH_ASSESS_EVENT = "{\n" +
-            "  \"assessmentTs\": 1568891729576,\n" +
-            "  \"batchId\": \"012846671379595264119\",\n" +
-            "  \"courseId\": \"do_2128415652377067521127\",\n" +
-            "  \"userId\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-            "  \"attemptId\": \"8cd87e24df268ad09a8b0060c0a40271\",\n" +
-            "  \"contentId\": \"do_212686723743318016173\",\n" +
-            "  \"events\": [\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1568891735461,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:db00a858fec1b8796c62f224874c7edf\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124784842112040965\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-            "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128415652377067521127\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"012846671379595264119\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124784842112040965\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_212686723743318016173\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128415652377067521127\",\n" +
-            "          \"l2\": \"do_2128415660716359681128\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124784842112040965\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"What is the Full form of WHO..?\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [],\n" +
-            "        \"duration\": 2\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1568891738245,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124784842112040965\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-            "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128415652377067521127\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"012846671379595264119\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124784842112040965\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_212686723743318016173\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128415652377067521127\",\n" +
-            "          \"l2\": \"do_2128415660716359681128\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124784842112040965\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"What is the Full form of WHO..?\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 4\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1568891747395,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:6ba5953669ea86e8f85759d3e7f5998b\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124784842112040965\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-            "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128415652377067521127\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"012846671379595264119\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124784842112040965\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_212686723743318016173\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128415652377067521127\",\n" +
-            "          \"l2\": \"do_2128415660716359681128\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124784842112040965\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"What is the Full form of WHO..?\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 1,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 14\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1568891772964,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:018f01bf99288474860b630b513b9d0c\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124784842112040965\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-            "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128415652377067521127\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"012846671379595264119\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124784842112040965\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_212686723743318016173\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128415652377067521127\",\n" +
-            "          \"l2\": \"do_2128415660716359681128\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124784842112040965\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"2bc922e7-985e-486a-ae23-4ba9a1c67edc\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mtf\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"lhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"}]\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"rhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"}]\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"lhs\\\":[\\\"1\\\",\\\"2\\\",\\\"3\\\"],\\\"rhs\\\":[\\\"3\\\",\\\"1\\\",\\\"2\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"MTF 3\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 2,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0.33,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"lhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"}]\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"rhs\": \"[{\\\"1\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"3\\\\\\\"}\\\"},{\\\"2\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"2\\\\\\\"}\\\"},{\\\"3\\\":\\\"{\\\\\\\"text\\\\\\\":\\\\\\\"1\\\\\\\"}\\\"}]\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 24\n" +
-            "      }\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-
-    public static final String BATCH_ASSESS__OLDER_EVENT = "{\n" +
-            "  \"courseId\": \"do_312712196780204032110117\",\n" +
-            "  \"batchId\": \"01271220181664563270\",\n" +
-            "  \"contentId\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "  \"userId\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "  \"assessmentTs\": 1567073236195,\n" +
-            "  \"attemptId\": \"attempt1\",\n" +
-            "  \"events\": [\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1567073236195,\n" +
-            "      \"ver\": \"3.0\",\n" +
-            "      \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"prod.diksha.portal\",\n" +
-            "          \"ver\": \"2.2.1\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" +
-            "        \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"01271220181664563270\",\n" +
-            "            \"type\": \"batch\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"do_312712196780204032110117\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_312592741863645184122936\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"3\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_3126430145280819201402\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"do_312468066279276544217372\",\n" +
-            "          \"maxscore\": 10,\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"TNXMATHS-STATISTICS 5\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"For any collection of n  items  ∑(χ - χ̅  )=\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 5,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"∑χ\": \"true\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 9\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1567073236195,\n" +
-            "      \"ver\": \"3.0\",\n" +
-            "      \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"prod.diksha.portal\",\n" +
-            "          \"ver\": \"2.2.1\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" +
-            "        \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"01271220181664563270\",\n" +
-            "            \"type\": \"batch\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"do_312712196780204032110117\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_312592741863645184122936\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"3\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_3126430145280819201402\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"do_312468066279276544217372\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"TNXMATHS-STATISTICS 5\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"For any collection of n  items  ∑(χ - χ̅  )=\"\n" +
-            "        },\n" +
-            "        \"index\": 2,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"∑χ\": \"true\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 9\n" +
-            "      }\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-
-    public static final String BATCH_ASSESS_FAIL_EVENT = "{\n" +
-            "  \"courseId\": \"do_312712196780204032110117\",\n" +
-            "  \"batchId\": \"01271220181664563270\",\n" +
-            "  \"contentId\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "  \"userId\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "  \"assessmentTs\": 1567073236195,\n" +
-            "  \"attemptId\": \"attempt1\",\n" +
-            "  \"events\": [\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1567687351000,\n" +
-            "      \"ver\": \"3.0\",\n" +
-            "      \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      }\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"prod.diksha.portal\",\n" +
-            "          \"ver\": \"2.2.1\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" +
-            "        \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"01271220181664563270\",\n" +
-            "            \"type\": \"batch\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"do_312712196780204032110117\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_312592741863645184122936\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"3\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_3126430145280819201402\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"do_312468066279276544217372\",\n" +
-            "          \"maxscore\": 10,\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"TNXMATHS-STATISTICS 5\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"For any collection of n  items  ∑(χ - χ̅  )=\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 5,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"∑χ\": \"true\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 9\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1567073236195,\n" +
-            "      \"ver\": \"3.0\",\n" +
-            "      \"mid\": \"ASSESS:48d754770446994b98e64577683ada25\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"prod.diksha.portal\",\n" +
-            "          \"ver\": \"2.2.1\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"310f123e-2764-c784-803a-0ca871f2b651\",\n" +
-            "        \"did\": \"b3541347e18ab916c06ed76aeb0ce57f\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"6b8617ccf0b4c35f3fe17ccdb71af908\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"01271220181664563270\",\n" +
-            "            \"type\": \"batch\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"do_312712196780204032110117\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_312592741863645184122936\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"3\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_3126430145280819201402\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"do_312468066279276544217372\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"TNXMATHS-STATISTICS 5\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"For any collection of n  items  ∑(χ - χ̅  )=\"\n" +
-            "        },\n" +
-            "        \"index\": 2,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"∑χ\": \"true\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 9\n" +
-            "      }\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-
-    public static final String BATCH_DUPLICATE_QUESTION_EVENT ="{\n" +
-            "  \"assessmentTs\": 1569304757079,\n" +
-            "  \"batchId\": \"01284169026368307244\",\n" +
-            "  \"courseId\": \"do_2128410273679114241112\",\n" +
-            "  \"userId\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "  \"attemptId\": \"90e1a0d12542806389a1a52aaf1fc622\",\n" +
-            "  \"contentId\": \"do_2128373396098744321673\",\n" +
-            "  \"events\": [\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304758743,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:5b2e689446886f3cee13de44fec8c02f\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 2\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304761105,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:0cba50019b880fc064a343a6d01d3d1a\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 4\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304763323,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:ace9f977fdb8254e709180777ff81ba6\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 6\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304765515,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:b4b19e35aed5b32cf240650aa09ec558\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"b7886294-95dd-4c83-91e3-7b67e82aaab2\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"normal distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"binomial distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"uniform distribution\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"3\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"In the textile industry, a manufacturer is interested in the number of blemishes or flaws occurring in each 100 feet of material. The probability distribution that has the greatest chance of applying to this situation is the\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 1,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"3\": \"{\\\"text\\\":\\\"Poisson distribution\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 8\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304767576,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:366e140cf5fddf850d1e548644a35729\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"759b8138-4c47-4724-8cd6-cae6a7d1ad18\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"descriptive statistic\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"probability function\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"4\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"A numerical description of the outcome of an experiment is called a\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 2,\n" +
-            "        \"pass\": \"No\",\n" +
-            "        \"score\": 0,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 1\n" +
-            "      }\n" +
-            "    },\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1569304769433,\n" +
-            "      \"ver\": \"3.1\",\n" +
-            "      \"mid\": \"ASSESS:834b83c5205b657a208e552d5964a872\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"d0d8a341-9637-484c-b871-0c27015af238\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0124511394914140160\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"staging.diksha.portal\",\n" +
-            "          \"ver\": \"2.4.0\",\n" +
-            "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"2cPynWPvF_X0xSqeCUOzka-kXEDT0vvw\",\n" +
-            "        \"did\": \"609b1be929adff933abd1b32caf10b6d\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"do_2128410273679114241112\",\n" +
-            "            \"type\": \"course\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"type\": \"batch\",\n" +
-            "            \"id\": \"01284169026368307244\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"f16a4cacf105ca65e98c61f5a63b8bd3\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0124511394914140160\",\n" +
-            "          \"l2\": \"01245115225042944040\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_2128373396098744321673\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_2128410273679114241112\",\n" +
-            "          \"l2\": \"do_2128410274404106241113\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [\n" +
-            "        \"0124511394914140160\",\n" +
-            "        \"01245115225042944040\"\n" +
-            "      ],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"759b8138-4c47-4724-8cd6-cae6a7d1ad18\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"type\": \"mcq\",\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"1\": \"{\\\"text\\\":\\\"descriptive statistic\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"2\": \"{\\\"text\\\":\\\"probability function\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"3\": \"{\\\"text\\\":\\\"variance\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "              \"answer\": \"{\\\"correct\\\":[\\\"4\\\"]}\"\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"title\": \"A numerical description of the outcome of an experiment is called a\\n\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 2,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 1,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"4\": \"{\\\"text\\\":\\\"random variable\\\\n\\\"}\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 3\n" +
-            "      }\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-    public static final String QUESTION_EVENT_RES_VALUES = "{\n" +
-            "  \"assessmentTs\": 1578663044164,\n" +
-            "  \"userId\": \"50a9e3fc-d047-4fa5-a37b-67501b8933db\",\n" +
-            "  \"contentId\": \"do_3129323935897108481169\",\n" +
-            "  \"courseId\": \"do_3129323995959541761169\",\n" +
-            "  \"batchId\": \"0129324118211215362\",\n" +
-            "  \"attemptId\": \"702ae8b81e37c94448d1fa117678d68c\",\n" +
-            "  \"events\": [\n" +
-            "    {\n" +
-            "      \"eid\": \"ASSESS\",\n" +
-            "      \"ets\": 1578663044164,\n" +
-            "      \"ver\": \"3.0\",\n" +
-            "      \"mid\": \"ASSESS:0b5c1d06920337df9d662b69d53f48bf\",\n" +
-            "      \"actor\": {\n" +
-            "        \"id\": \"50a9e3fc-d047-4fa5-a37b-67501b8933db\",\n" +
-            "        \"type\": \"User\"\n" +
-            "      },\n" +
-            "      \"context\": {\n" +
-            "        \"channel\": \"0126684405014528002\",\n" +
-            "        \"pdata\": {\n" +
-            "          \"id\": \"prod.diksha.app\",\n" +
-            "          \"ver\": \"2.6.203\",\n" +
-            "          \"pid\": \"sunbird.app.contentplayer\"\n" +
-            "        },\n" +
-            "        \"env\": \"contentplayer\",\n" +
-            "        \"sid\": \"11059dcc-0a69-4d92-90b3-c1eb97f5f93b\",\n" +
-            "        \"did\": \"b9bdcb8cd7abc5bd7813bd65ec0b5084dc0dadd8\",\n" +
-            "        \"cdata\": [\n" +
-            "          {\n" +
-            "            \"id\": \"83f8c374dc0e4d91ec4c48d6e5d10b4c\",\n" +
-            "            \"type\": \"AttemptId\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"streaming\",\n" +
-            "            \"type\": \"PlayerLaunch\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"6a7809b4eccf75ae62b18c224a4cfeb5\",\n" +
-            "            \"type\": \"ContentSession\"\n" +
-            "          },\n" +
-            "          {\n" +
-            "            \"id\": \"48e8e20e8c6fbc660f4ca317497fa7a6\",\n" +
-            "            \"type\": \"PlaySession\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"0126684405014528002\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"object\": {\n" +
-            "        \"id\": \"do_3129323935897108481169\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"rollup\": {\n" +
-            "          \"l1\": \"do_3129323995959541761169\",\n" +
-            "          \"l2\": \"do_31293239986759270411495\"\n" +
-            "        }\n" +
-            "      },\n" +
-            "      \"tags\": [],\n" +
-            "      \"edata\": {\n" +
-            "        \"item\": {\n" +
-            "          \"id\": \"QMCQ02115\",\n" +
-            "          \"maxscore\": 1,\n" +
-            "          \"exlength\": 0,\n" +
-            "          \"params\": [\n" +
-            "            {\n" +
-            "              \"item\": {\n" +
-            "                \"keywords\": [\n" +
-            "                  \"mdd\"\n" +
-            "                ],\n" +
-            "                \"subject\": \"Mathematics\",\n" +
-            "                \"channel\": \"in.ekstep\",\n" +
-            "                \"language\": [\n" +
-            "                  \"English\"\n" +
-            "                ],\n" +
-            "                \"type\": \"ftb\",\n" +
-            "                \"editorState\": null,\n" +
-            "                \"body\": null,\n" +
-            "                \"question_audio\": \"\",\n" +
-            "                \"gradeLevel\": [\n" +
-            "                  \"Class 4\"\n" +
-            "                ],\n" +
-            "                \"appId\": \"prod.diksha.app\",\n" +
-            "                \"used_for\": \"worksheet\",\n" +
-            "                \"model\": {\n" +
-            "                  \"hintMsg\": \"Observe the key value and find the number of aircrafts which flies in the morning.\",\n" +
-            "                  \"numericLangId\": \"en\",\n" +
-            "                  \"langId\": \"en\",\n" +
-            "                  \"variables\": {\n" +
-            "                    \"$aircraft1\": \"random(100,100)\",\n" +
-            "                    \"$aircraft2\": \"random(70,70)\",\n" +
-            "                    \"$aircraft3\": \"random(80,80)\",\n" +
-            "                    \"$aircraft4\": \"random(30,30)\"\n" +
-            "                  },\n" +
-            "                  \"dropDowns\": [\n" +
-            "                    {\n" +
-            "                      \"identifier\": 1,\n" +
-            "                      \"options\": [\n" +
-            "                        {\n" +
-            "                          \"text\": \"OPT_1_0\",\n" +
-            "                          \"mh\": null,\n" +
-            "                          \"mmc\": [\n" +
-            "                            \"DH277\",\n" +
-            "                            \"DH278\"\n" +
-            "                          ],\n" +
-            "                          \"answer\": true,\n" +
-            "                          \"image\": null\n" +
-            "                        },\n" +
-            "                        {\n" +
-            "                          \"text\": \"OPT_1_1\",\n" +
-            "                          \"mh\": \"MH_1_1\",\n" +
-            "                          \"mmc\": [\n" +
-            "                            \"DH277\",\n" +
-            "                            \"DH278\"\n" +
-            "                          ],\n" +
-            "                          \"answer\": false,\n" +
-            "                          \"image\": null\n" +
-            "                        },\n" +
-            "                        {\n" +
-            "                          \"text\": \"OPT_1_2\",\n" +
-            "                          \"mh\": \"MH_1_2\",\n" +
-            "                          \"mmc\": [\n" +
-            "                            \"DH277\",\n" +
-            "                            \"DH278\"\n" +
-            "                          ],\n" +
-            "                          \"answer\": false,\n" +
-            "                          \"image\": null\n" +
-            "                        },\n" +
-            "                        {\n" +
-            "                          \"text\": \"OPT_1_3\",\n" +
-            "                          \"mh\": \"MH_1_3\",\n" +
-            "                          \"mmc\": [\n" +
-            "                            \"DH277\",\n" +
-            "                            \"DH278\"\n" +
-            "                          ],\n" +
-            "                          \"answer\": false,\n" +
-            "                          \"image\": null\n" +
-            "                        }\n" +
-            "                      ]\n" +
-            "                    },\n" +
-            "                    {\n" +
-            "                      \"identifier\": 2,\n" +
-            "                      \"options\": []\n" +
-            "                    },\n" +
-            "                    {\n" +
-            "                      \"identifier\": 3,\n" +
-            "                      \"options\": []\n" +
-            "                    },\n" +
-            "                    {\n" +
-            "                      \"identifier\": 4,\n" +
-            "                      \"options\": []\n" +
-            "                    }\n" +
-            "                  ],\n" +
-            "                  \"variablesProcessed\": true\n" +
-            "                },\n" +
-            "                \"state\": \"Verified\",\n" +
-            "                \"identifier\": \"QMCQ02115\",\n" +
-            "                \"level\": 2,\n" +
-            "                \"author\": \"funtoot\",\n" +
-            "                \"consumerId\": \"1762141d-8681-458b-9bd1-c6af98c0d989\",\n" +
-            "                \"solutions\": null,\n" +
-            "                \"portalOwner\": \"562\",\n" +
-            "                \"version\": 1,\n" +
-            "                \"i18n\": {\n" +
-            "                  \"en\": {\n" +
-            "                    \"HINT\": \"Hint\",\n" +
-            "                    \"MICROHINT\": \"Micro Hint\",\n" +
-            "                    \"SOLUTION\": \"Solution\",\n" +
-            "                    \"HELP\": \"Help\",\n" +
-            "                    \"REDUCE_FRACTION\": \"Check the solution again. Remove the common factors in the fraction and reduce it to its lowest term.\",\n" +
-            "                    \"MIXED_FRACTION\": \"Convert the mixed fraction to an improper fraction.\",\n" +
-            "                    \"IMPROPER_FRACTION\": \"Convert the improper fraction to an mixed fraction.\",\n" +
-            "                    \"LIKEFRACTIONADDITION_NUMERATOR\": \"Add the numerator of the given fractions correctly.\",\n" +
-            "                    \"LIKEFRACTIONADDITION_DENOMINATOR\": \"For addition of like fractions the denominator remains the same.\",\n" +
-            "                    \"LIKEFRACTIONADDITION_FULL\": \"Check the addition of the numerator. The denominator remains the same.\",\n" +
-            "                    \"UNLIKEFRACTIONADDITION_NUMERATOR\": \"First convert the unlike fractions to like fractions. Then add the numerator of the like fractions.\",\n" +
-            "                    \"UNLIKEFRACTIONADDITION_DENOMINATOR\": \"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.\",\n" +
-            "                    \"UNLIKEFRACTIONADDITION_FULL\": \"Convert the given fractions to like fractions and then carefully do the addition.\",\n" +
-            "                    \"LIKEFRACTIONSUBTRACTION_NUMERATOR\": \"Subtract the numerator of the given fractions correctly.\",\n" +
-            "                    \"LIKEFRACTIONSUBTRACTION_DENOMINATOR\": \"For subtraction of like fractions the denominator remains the same as that of the question.\",\n" +
-            "                    \"LIKEFRACTIONSUBTRACTION_FULL\": \"Subtract the numerator of the fractions. Denominator remains the same.\",\n" +
-            "                    \"UNLIKEFRACTIONSUBTRACTION_NUMERATOR\": \"First convert the unlike fractions to like fractions. Then subtract the numerator of the like fractions.\",\n" +
-            "                    \"UNLIKEFRACTIONSUBTRACTION_DENOMINATOR\": \"First find the least common factor and convert the unlike fractions to like fractions. The denominator of the solution is the denominator of the like fractions.\",\n" +
-            "                    \"UNLIKEFRACTIONSUBTRACTION_FULL\": \"Convert the given fractions to like fractions and then carefully do the subtraction.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONADDITION_WHOLE\": \"Add the whole number part of both the fractions carefully. Check for the conversion of the improper fraction to proper fractions in the solution.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONADDITION_NUMERATOR\": \"Add the numerator of the like fractions. If the fractional part is an improper fraction, then convert it to a proper fraction.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONADDITION_DENOMINATOR\": \"The denominator of the solution fraction does not change for like fractions.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONADDITION_FULL\": \"Convert the mixed fractions to improper fractions and then add them. The final solution should again be as a mixed fraction.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONSUBTRACTION_WHOLE\": \"Subtract the whole number part of both the fractions carefully.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONSUBTRACTION_NUMERATOR\": \"Subtract the numerator of the like fractions.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONSUBTRACTION_DENOMINATOR\": \"The denominator of the solution fraction does not change for like fractions.\",\n" +
-            "                    \"LIKEMIXEDFRACTIONSUBTRACTION_FULL\": \"Convert the mixed fractions to improper fractions and then subtract them. The final solution should again be as a mixed fraction.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONADDITION_WHOLE\": \"The integer added becomes the whole number part of the mixed fraction.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONADDITION_NUMERATOR\": \"The numerator of the solution fraction remains the same. \",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONADDITION_DENOMINATOR\": \"The denominator of the solution fraction remains the same as given in the problem.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONADDITION_FULL\": \"The integer and the proper fraction when added becomes the mixed fraction.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONSUBTRACTION_WHOLE\": \"Convert the whole number to an equivalent fraction. Then find the difference between the like fractions.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONSUBTRACTION_NUMERATOR\": \"First find the least common factor and convert them to difference between like fractions. Then subtract the numerators carefully.\",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONSUBTRACTION_DENOMINATOR\": \"The denominator of the solution fraction remains the same as given in the problem. \",\n" +
-            "                    \"WHOLEANDPROPERFRACTIONSUBTRACTION_FULL\": \"When the proper fraction is subtracted from the integer becomes the mixed fraction.\",\n" +
-            "                    \"WHOLEANDFRACTIONMULTIPLICATION_NUMERATOR\": \"Multiply the numerator and the whole number carefully.\",\n" +
-            "                    \"WHOLEANDFRACTIONMULTIPLICATION_DENOMINATOR\": \"Multiply the denominators of the fractions carefully. \",\n" +
-            "                    \"WHOLEANDFRACTIONMULTIPLICATION_FULL\": \"Multiply the integer with the fraction.\",\n" +
-            "                    \"FRACTIONMULTIPLICATION_NUMERATOR\": \"Multiply the numerators of the fractions carefully.\",\n" +
-            "                    \"FRACTIONMULTIPLICATION_DENOMINATOR\": \"Multiply the denominators of the fractions carefully. \",\n" +
-            "                    \"FRACTIONMULTIPLICATION_FULL\": \"Multiply the numerators and denominators of the fractions.\",\n" +
-            "                    \"FRACTIONDIVISION\": \"First find the reciprocal of the divisor. Then multiply the fraction with the reciprocal of the divisor to get the solution.\",\n" +
-            "                    \"SELECT\": \"Select\",\n" +
-            "                    \"EXPRESSIONS\": \"$aircraft1=random(100,100)\\n$aircraft2=random(70,70)\\n$aircraft3=random(80,80)\\n$aircraft4=random(30,30)\",\n" +
-            "                    \"NO_HINT\": \"There is no hint for this question.\",\n" +
-            "                    \"NO_ANSWER\": \"Please answer.\",\n" +
-            "                    \"SOLUTION_ID\": \"a. The tallest bar shows the subject which is most liked by the students i.e., Social Studies. 40 students like Social Studiesb. There are 5 subjects mentioned in the graph, namely English, Mathematics, Hindi, Social Studies, and General Science.c. The shortest bar represents the subject which is least liked by the students i.e., Mathematics. Only 10 students like Mathematics.d. Number of students who like Hindi = 20Number of students who like Mathematics = 10We can clearly see that the number of students who like Hindi is double  that of Mathematics. Hence the statement is true.\",\n" +
-            "                    \"HINT_ID\": \"Observe the key value and find the number of aircrafts which flies in the morning.\",\n" +
-            "                    \"QUESTION_TEXT\": \"The pictograph represents the number of aircrafts flying from India to foreign countries at different timings. Airlines decided to fly $aircraft1 aircrafts in the morning. How many aircrafts should they fly more? __1__\",\n" +
-            "                    \"MH_1_3\": \"Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" +
-            "                    \"OPT_1_3\": \"$aircraft1\",\n" +
-            "                    \"MH_1_2\": \"According to the pictograph the number of aircrafts which flies in the afternoon is $aircraft3. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" +
-            "                    \"OPT_1_2\": \"$aircraft3\",\n" +
-            "                    \"MH_1_1\": \"According to the pictograph the number of aircrafts which flies in the morning is $aircraft2. Find how many number of aircrafts to be added in the morning to make the count as $aircraft1.\",\n" +
-            "                    \"OPT_1_1\": \"$aircraft2\",\n" +
-            "                    \"OPT_1_0\": \"$aircraft4\"\n" +
-            "                  }\n" +
-            "                },\n" +
-            "                \"tags\": [\n" +
-            "                  \"mdd\"\n" +
-            "                ],\n" +
-            "                \"concepts\": [\n" +
-            "                  \"C421\"\n" +
-            "                ],\n" +
-            "                \"grade\": [\n" +
-            "                  \"4\"\n" +
-            "                ],\n" +
-            "                \"domain\": \"Numeracy\",\n" +
-            "                \"name\": \"QMCQ02115\",\n" +
-            "                \"bloomsTaxonomyLevel\": \"Understand\",\n" +
-            "                \"status\": \"Live\",\n" +
-            "                \"itemType\": \"UNIT\",\n" +
-            "                \"code\": \"QMCQ02115\",\n" +
-            "                \"qtype\": \"mdd\",\n" +
-            "                \"qlevel\": \"MEDIUM\",\n" +
-            "                \"flags\": \"{\\\"isZoomable\\\":true}\",\n" +
-            "                \"questionImage\": \"org.ekstep.funtoot.QMCQ02115.image.9074280266192281\",\n" +
-            "                \"media\": [\n" +
-            "                  {\n" +
-            "                    \"id\": \"org.ekstep.funtoot.QMCQ02115.image.9074280266192281\",\n" +
-            "                    \"src\": \"https://ekstep-public-prod.s3-ap-south-1.amazonaws.com/content/org.ekstep.funtoot.qmcq02115.image.9886103695997586/artifact/org.ekstep.funtoot.qmcq02115.image.9886103695997586_1514092203316.png\",\n" +
-            "                    \"type\": \"image\"\n" +
-            "                  }\n" +
-            "                ],\n" +
-            "                \"title\": \"\",\n" +
-            "                \"qid\": \"QMCQ02115\",\n" +
-            "                \"createdOn\": \"2017-12-24T05:10:03.956+0000\",\n" +
-            "                \"qindex\": 1,\n" +
-            "                \"lastUpdatedOn\": \"2018-08-16T14:08:14.323+0000\",\n" +
-            "                \"subLevel\": \"\",\n" +
-            "                \"question\": \"QUESTION_TEXT\",\n" +
-            "                \"versionKey\": \"1534428494323\",\n" +
-            "                \"framework\": \"NCF\",\n" +
-            "                \"answer\": {},\n" +
-            "                \"max_score\": 5,\n" +
-            "                \"sublevel\": 3,\n" +
-            "                \"template_id\": \"org.ekstep.funtoot.template.01\",\n" +
-            "                \"category\": \"MCQ\",\n" +
-            "                \"isSelected\": true,\n" +
-            "                \"$$hashKey\": \"object:1540\",\n" +
-            "                \"template\": \"funtoot.template.01\",\n" +
-            "                \"maxAttempts\": 2,\n" +
-            "                \"curAttempt\": 0\n" +
-            "              },\n" +
-            "              \"config\": {\n" +
-            "                \"count\": 1,\n" +
-            "                \"selectedConfig\": {},\n" +
-            "                \"title\": \"\",\n" +
-            "                \"type\": \"items\",\n" +
-            "                \"var\": \"item\"\n" +
-            "              }\n" +
-            "            }\n" +
-            "          ],\n" +
-            "          \"uri\": \"\",\n" +
-            "          \"mmc\": [],\n" +
-            "          \"mc\": [\n" +
-            "            null\n" +
-            "          ],\n" +
-            "          \"desc\": \"\"\n" +
-            "        },\n" +
-            "        \"index\": 1,\n" +
-            "        \"pass\": \"Yes\",\n" +
-            "        \"score\": 1,\n" +
-            "        \"resvalues\": [\n" +
-            "          {\n" +
-            "            \"1\": \"random(30,30)\"\n" +
-            "          }\n" +
-            "        ],\n" +
-            "        \"duration\": 6\n" +
-            "      }\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-
-    public static Map<String, Object> getMap(String message) {
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java b/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java
deleted file mode 100644
index a47e2334e8..0000000000
--- a/data-pipeline/assessment-aggregator/src/test/java/org/ekstep/ep/samza/task/AssessmentAggregatorTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.UDTValue;
-import com.datastax.driver.core.UserType;
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Aggregate;
-import org.ekstep.ep.samza.domain.BatchEvent;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.service.AssessmentAggregatorService;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.util.Date;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
-
-public class AssessmentAggregatorTest {
-
-    private IncomingMessageEnvelope envelope;
-    private MessageCollector collector;
-    private AssessmentAggregatorTask assessmentAggregatorTask;
-    private TaskCoordinator taskCoordinator;
-    private TaskContext taskContext;
-    private CassandraConnect cassandraConnect;
-    private AssessmentAggregatorConfig config;
-    private Counter counter;
-
-    @Before
-    public void setUp() {
-        Config configMock = mock(Config.class);
-        envelope = mock(IncomingMessageEnvelope.class);
-        collector = mock(MessageCollector.class);
-        taskCoordinator = mock(TaskCoordinator.class);
-        taskContext = mock(TaskContext.class);
-        MetricsRegistry metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        stub(metricsRegistry.newCounter(anyString(), anyString()))
-                .toReturn(counter);
-        stub(taskContext.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(envelope.getOffset()).toReturn("2");
-        stub(envelope.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1)));
-        stub(configMock.get("middleware.cassandra.host", "127.0.0.1")).toReturn("127.0.0.1");
-        stub(configMock.get("middleware.cassandra.port", "9042")).toReturn("9042");
-        stub(configMock.get("middleware.cassandra.courses_keyspace", "sunbird_courses")).toReturn("sunbird_courses");
-        stub(configMock.get("middleware.cassandra.aggregator_table", "assessment_aggregator")).toReturn("assessment_aggregator");
-        stub(configMock.get("middleware.cassandra.question_type", "question")).toReturn("question");
-        stub(configMock.get("output.failed.topic.name", "telemetry.assess.failed")).toReturn("telemetry.assess.failed");
-        UserType userType = mock(UserType.class);
-        UDTValue udtValue = mock(UDTValue.class);
-        when(udtValue.setString(anyString(), anyString())).thenReturn(udtValue);
-        when(udtValue.setString(any(), any())).thenReturn(udtValue);
-        when(udtValue.setDouble(anyString(), anyDouble())).thenReturn(udtValue);
-        when(udtValue.setDecimal(anyString(), any())).thenReturn(udtValue);
-        when(udtValue.setList(anyString(), anyList())).thenReturn(udtValue);
-        when(udtValue.setTimestamp(anyString(), any())).thenReturn(udtValue);
-        when(userType.newValue()).thenReturn(udtValue);
-        cassandraConnect = mock(CassandraConnect.class);
-        when(cassandraConnect.getUDTType("sunbird_courses", "question")).thenReturn(userType);
-        config = new AssessmentAggregatorConfig(configMock);
-        assessmentAggregatorTask = new AssessmentAggregatorTask(configMock, taskContext, cassandraConnect);
-
-    }
-
-    @Test
-    public void shouldUpdateCassandra() throws Exception {
-
-        String event = EventFixture.BATCH_ASSESS_EVENT;
-        BatchEvent batchEvent = new BatchEvent((Map<String, Object>) new Gson().fromJson(event, Map.class));
-        stub(envelope.getMessage()).toReturn(event);
-        assessmentAggregatorTask.process(envelope, collector, taskCoordinator);
-        AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService
-                (cassandraConnect, config);
-        AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector,
-                new JobMetrics(taskContext, "AssessmentAggregator"), config);
-        Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, 1565198476000L);
-        assertEquals(2.0, assess.getTotalMaxScore(), 0.001);
-        assertEquals(1.33, assess.getTotalScore(), 0.001);
-        assertEquals("1.33/2.0", assess.getGrandTotal());
-    }
-
-    @Test
-    public void shouldSkipBatchIfAssessmentDateIsOlder() throws Exception {
-
-        String event = EventFixture.BATCH_ASSESS__OLDER_EVENT;
-        stub(envelope.getMessage()).toReturn(event);
-        Row row = mock(Row.class);
-        stub(row.getTimestamp("last_attempted_on")).toReturn(new Date(1567444876000L));
-        when(cassandraConnect.findOne(anyString())).thenReturn(row);
-        assessmentAggregatorTask.process(envelope, collector, taskCoordinator);
-        verify(collector).send(argThat(validateOutputTopic(envelope.getMessage(), "telemetry.assess.failed")));
-        verify(counter, times(2)).inc();
-    }
-
-    @Test
-    public void shouldSendDuplicateBatchEventToFailedTopic() throws Exception {
-        stub(envelope.getMessage()).toReturn(EventFixture.BATCH_ASSESS_FAIL_EVENT);
-        AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService(cassandraConnect, config);
-        AssessmentAggregatorSource assessmentAggregatorSource = new AssessmentAggregatorSource(envelope);
-        AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector, mock(JobMetrics.class), config);
-        assessmentAggregatorService.process(assessmentAggregatorSource, sink);
-        verify(collector).send(argThat(validateOutputTopic(envelope.getMessage(), "telemetry.assess.failed")));
-    }
-
-    @Test
-    public void shouldTakeLatestAssessEventInBatchWhenDuplicates() throws Exception {
-        String event = EventFixture.BATCH_DUPLICATE_QUESTION_EVENT;
-        BatchEvent batchEvent = new BatchEvent((Map<String, Object>) new Gson().fromJson(event, Map.class));
-        stub(envelope.getMessage()).toReturn(event);
-        AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService
-                (cassandraConnect, config);
-        Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, 1565198476000L);
-        assertEquals(2, assess.getQuestionsList().size());
-        assertEquals(2.0, assess.getTotalMaxScore(), 0.001);
-        assertEquals(2.0, assess.getTotalScore(), 0.001);
-
-    }
-
-    @Test
-    public void shouldUpdateCassandraWithArrayValues() throws Exception {
-
-        String event = EventFixture.QUESTION_EVENT_RES_VALUES;
-        BatchEvent batchEvent = new BatchEvent((Map<String, Object>) new Gson().fromJson(event, Map.class));
-        stub(envelope.getMessage()).toReturn(event);
-        assessmentAggregatorTask.process(envelope, collector, taskCoordinator);
-        AssessmentAggregatorService assessmentAggregatorService = new AssessmentAggregatorService
-                (cassandraConnect, config);
-        AssessmentAggregatorSink sink = new AssessmentAggregatorSink(collector,
-                new JobMetrics(taskContext, "AssessmentAggregator"), config);
-        Aggregate assess = assessmentAggregatorService.getAggregateData(batchEvent, new Date().getTime());
-        assertEquals(1.0, assess.getTotalMaxScore(), 0.001);
-        assertEquals(1.0, assess.getTotalScore(), 0.001);
-        assertEquals("1.0/1.0", assess.getGrandTotal());
-    }
-
-
-    private ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/assessment-aggregator/src/test/resources/test.cql b/data-pipeline/assessment-aggregator/src/test/resources/test.cql
deleted file mode 100644
index 80208aa495..0000000000
--- a/data-pipeline/assessment-aggregator/src/test/resources/test.cql
+++ /dev/null
@@ -1,35 +0,0 @@
-CREATE KEYSPACE sunbird_courses with replication = {'class':'SimpleStrategy','replication_factor':1};
-
-
-CREATE TYPE IF NOT EXISTS sunbird_courses.question (
-id text,
-assess_ts timestamp,
-max_score double,
-score double,
-type text,
-title text,
-resvalues list<frozen<map<text, text>>>,
-params list<frozen<map<text, text>>>,
-description text,
-duration decimal
-);
-
-
-CREATE TABLE IF NOT EXISTS sunbird_courses.assessment_aggregator (
-course_id text,
-batch_id text,
-content_id text,
-attempt_id text,
-user_id text,
-created_on timestamp,
-last_attempted_on timestamp,
-total_max_score double,
-grand_total text,
-question list<frozen<question>>,
-total_score double,
-updated_on timestamp,
-PRIMARY KEY (course_id, batch_id, content_id, attempt_id, user_id)
-);
-
-CREATE INDEX assessment_aggregator_last_attempted_on_idx ON sunbird_courses.assessment_aggregator (last_attempted_on);
-
diff --git a/data-pipeline/content-cache-updater/pom.xml b/data-pipeline/content-cache-updater/pom.xml
deleted file mode 100644
index 06e93f359c..0000000000
--- a/data-pipeline/content-cache-updater/pom.xml
+++ /dev/null
@@ -1,174 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>jobs</artifactId>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <version>0.0.1</version>
-    </parent>
-
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>content-cache-updater</artifactId>
-    <version>0.0.4</version>
-    <name>ContentCacheUpdater</name>
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
-            <version>RELEASE</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.squareup.okhttp3</groupId>
-            <artifactId>okhttp</artifactId>
-            <version>4.4.0</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file
diff --git a/data-pipeline/content-cache-updater/src/main/assembly/src.xml b/data-pipeline/content-cache-updater/src/main/assembly/src.xml
deleted file mode 100644
index ca9b9f29f9..0000000000
--- a/data-pipeline/content-cache-updater/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/content-cache-updater.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:content-cache-updater</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-                <include>redis.clients:2.9.0</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties b/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties
deleted file mode 100644
index f3f51ffd00..0000000000
--- a/data-pipeline/content-cache-updater/src/main/config/content-cache-updater.properties
+++ /dev/null
@@ -1,89 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.ContentCacheUpdater
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__redis_updater_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.ContentCacheUpdaterTask
-task.inputs=kafka.__env__.learning.graph.events
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,skipped-message-count
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-
-# redis
-# redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=3
-redis.connection.idle.max=3
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.contentStore.id=5
-redis.database.dialCodeStore.id=6
-
-# content model fields to convert to List from String
-contentModel.fields.listType=gradeLevel,subject,medium,language
-date.fields.listType=lastStatusChangedOn,lastUpdatedOn,createdOn
-
-dialcode.api.host = __dialcode_host__
-dialcode.api.endpoint = __dialcode_api_endpoint__
-dialcode.api.authorizationkey =__dialcode_authorization_key__
-
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java
deleted file mode 100644
index 503dffd87c..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/service/ContentCacheUpdaterService.java
+++ /dev/null
@@ -1,120 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.core.BaseCacheUpdaterService;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.task.ContentCacheConfig;
-import org.ekstep.ep.samza.task.ContentCacheUpdaterSink;
-import org.ekstep.ep.samza.task.ContentCacheUpdaterSource;
-import org.ekstep.ep.samza.util.ContentData;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.util.RestUtil;
-
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class ContentCacheUpdaterService extends BaseCacheUpdaterService {
-
-    private static Logger LOGGER = new Logger(ContentCacheUpdaterService.class);
-    private JobMetrics metrics;
-    private ContentData contentData;
-    private ContentCacheConfig contentCacheConfig;
-    private RestUtil restUtil;
-    int storeId;
-    private Gson gson = new Gson();
-    Type mapType = new TypeToken<Map<String, Object>>() {
-    }.getType();
-
-    public ContentCacheUpdaterService(ContentCacheConfig config, RedisConnect redisConnect, JobMetrics metrics, RestUtil restUtil) {
-        super(redisConnect);
-        this.metrics = metrics;
-        this.contentCacheConfig = config;
-        this.restUtil = restUtil;
-    }
-
-    public void process(ContentCacheUpdaterSource source, ContentCacheUpdaterSink sink) {
-        Map<String, Object> message = source.getMap();
-        String nodeUniqueId = (String) message.get("nodeUniqueId");
-        String objectType = (String) message.get("objectType");
-        Map<String, Object> parsedData = null;
-        if (nodeUniqueId == null || objectType == null || nodeUniqueId.isEmpty() || objectType.isEmpty()) {
-            sink.markSkipped();
-            return;
-        }
-        LOGGER.info("", "processing event for nodeUniqueId: " + nodeUniqueId);
-        contentData = new ContentData();
-        if (!nodeUniqueId.isEmpty()) {
-            parsedData = getCacheData(message);
-        }
-        if (null != parsedData) {
-            addToCache(nodeUniqueId, gson.toJson(parsedData), storeId);
-            LOGGER.info(nodeUniqueId, "Updated in cache for type " + objectType);
-            sink.success();
-        }
-    }
-
-    public Map<String, Object> getCacheData(Map<String, Object> message) {
-        String nodeUniqueId = (String) message.get("nodeUniqueId");
-        Map<String, Object> parsedData = null;
-        Map<String, Object> newProperties = contentData.extractProperties(message);
-        /**
-         * Responsibility:
-         * 1. Take dialCode property form the transaction event.
-         * 2. Check whether dialCode metadata is present in the cache or not.
-         * 3. If not present in the redis
-         *      1. Get the dialCode metadata using dialCode api and update to redis cache
-         */
-        updateDialCodeToCache((List<String>) newProperties.get("dialcodes"), contentCacheConfig.getdialCodeStoreDb(), contentCacheConfig.getDialCodeAPIUrl());
-        updateDialCodeToCache(getDialCodesAsList(gson.fromJson((String) newProperties.get("reservedDialcodes"), Map.class)), contentCacheConfig.getdialCodeStoreDb(), contentCacheConfig.getDialCodeAPIUrl());
-        String contentNode = null;
-        storeId = contentCacheConfig.getContentStoreDb();
-        contentNode = readFromCache(nodeUniqueId, storeId);
-        Map<String, Object> listTypeFields = contentData.convertType(newProperties, contentCacheConfig.getContentModelListTypeFields(), contentCacheConfig.getDateFields());
-        if (!listTypeFields.isEmpty()) {
-            newProperties.putAll(listTypeFields);
-        }
-        if (contentNode != null) {
-            parsedData = gson.fromJson(contentNode, mapType);
-        } else {
-            parsedData = new HashMap<>();
-        }
-        return contentData.getParsedData(parsedData, newProperties);
-    }
-
-    public void updateDialCodeToCache(List<String> dialCodeList, int storeId, String apiUrl) {
-        if (null != dialCodeList) {
-            dialCodeList.forEach(dialCode -> {
-                if (!dialCode.isEmpty()) {
-                    metrics.incDialCodesCount();
-                    String metaData = readFromCache(dialCode, storeId);
-                    if (null == metaData || metaData.isEmpty()) {
-                        LOGGER.info("", String.format("Invoking DialCode API to fetch the metadata for this dialCode( %s )", dialCode));
-                        Object dialCodeMetadata = contentData.getMetadata(apiUrl.concat(dialCode), this.restUtil, contentCacheConfig.getAuthorizationKey(), "dialcode");
-                        if (null != dialCodeMetadata) {
-                            metrics.incDialCodesFromApiCount();
-                            LOGGER.info("", String.format("Inserting dialCode( %s ) metadata to cache system", dialCode));
-                            addToCache(dialCode, gson.toJson(dialCodeMetadata), storeId);
-                        } else {
-                            LOGGER.info("", String.format("API did not fetched any dialCode metadata!!! %s", dialCode));
-                        }
-                    } else {
-                        metrics.incDialCodesFromCacheCount();
-                    }
-                }
-            });
-        }
-    }
-
-    public List<String> getDialCodesAsList(Map<String, Object> dialCodesMap) {
-        if (null != dialCodesMap) {
-            return new ArrayList<>(dialCodesMap.keySet());
-        } else {
-            return new ArrayList<>();
-        }
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java
deleted file mode 100644
index 6a1a8cdff7..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheConfig.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class ContentCacheConfig {
-
-    private final String JOB_NAME = "ContentCacheUpdater";
-
-    private int contentStoreDb;
-    private int dialCodeStoreDb;
-    private List<String> contentModelListTypeFields;
-    private List<String> dateFields;
-    private String apiHost;
-    private String dialCodeAPiEndPoint;
-    private String authorizationKey = "";
-
-    public ContentCacheConfig(Config config) {
-        this.contentStoreDb = config.getInt("redis.database.contentStore.id", 5);
-        this.dialCodeStoreDb = config.getInt("redis.database.dialCodeStore.id", 6);
-        this.contentModelListTypeFields = config.getList("contentModel.fields.listType", new ArrayList<>());
-        this.dateFields = config.getList("date.fields.listType", new ArrayList<>());
-        apiHost = config.get("dialcode.api.host","https://localhost");
-        dialCodeAPiEndPoint = config.get("dialcode.api.endpoint","/api/dialcode/v3/read/");
-        authorizationKey = config.get("dialcode.api.authorizationkey","");
-    }
-
-    public int getContentStoreDb() { return this.contentStoreDb; }
-
-    public int getdialCodeStoreDb() { return this.dialCodeStoreDb; }
-
-    public List<String> getContentModelListTypeFields() { return this.contentModelListTypeFields; }
-
-    public List<String> getDateFields() { return this.dateFields; }
-
-    public String JOB_NAME() { return JOB_NAME; }
-
-    public String getDialCodeAPIUrl(){
-        return apiHost.concat(dialCodeAPiEndPoint);
-    }
-    public String getAuthorizationKey(){
-        return this.authorizationKey;
-    }
-}
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java
deleted file mode 100644
index 2ca306a503..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSink.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-public class ContentCacheUpdaterSink extends BaseSink {
-
-    public ContentCacheUpdaterSink(MessageCollector collector, JobMetrics metrics) {
-        super(collector, metrics);
-    }
-
-    public void success() {
-        metrics.incSuccessCounter();
-    }
-
-    public void markSkipped() { metrics.incSkippedCounter(); }
-}
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java
deleted file mode 100644
index ac9032f6fe..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterSource.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-
-import java.util.Map;
-
-public class ContentCacheUpdaterSource {
-
-    private IncomingMessageEnvelope envelope;
-
-    public ContentCacheUpdaterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-}
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java
deleted file mode 100644
index c34e072e02..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/task/ContentCacheUpdaterTask.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.ContentCacheUpdaterService;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.util.RestUtil;
-
-public class ContentCacheUpdaterTask extends BaseSamzaTask {
-
-    private JobMetrics metrics;
-    private ContentCacheUpdaterService service;
-    private ContentCacheConfig config;
-    private RestUtil restUtil;
-
-
-    public ContentCacheUpdaterTask(Config config, TaskContext context, RedisConnect redisConnect, RestUtil restUtil) {
-        init(config, context, redisConnect, restUtil);
-    }
-
-    public ContentCacheUpdaterTask() {
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) {
-        init(config, context, null, restUtil);
-    }
-
-    public void init(Config config, TaskContext context, RedisConnect redisConnect, RestUtil restUtil) {
-        this.config = new ContentCacheConfig(config);
-        metrics = new JobMetrics(context, this.config.JOB_NAME());
-        redisConnect = null != redisConnect? redisConnect: new RedisConnect(config);
-        this.restUtil = null != restUtil ? restUtil : new RestUtil();
-        service = new ContentCacheUpdaterService(this.config, redisConnect, metrics, this.restUtil);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelope);
-        ContentCacheUpdaterSink sink = new ContentCacheUpdaterSink(collector, metrics);
-        service.process(source, sink);
-    }
-    
-}
diff --git a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java b/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java
deleted file mode 100644
index abe3a96516..0000000000
--- a/data-pipeline/content-cache-updater/src/main/java/org/ekstep/ep/samza/util/ContentData.java
+++ /dev/null
@@ -1,81 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import kong.unirest.UnirestException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.task.ContentCacheConfig;
-
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-public class ContentData {
-    static Logger LOGGER = new Logger(ContentData.class);
-
-    public Map<String, Object> getParsedData(Map<String, Object> parsedData, Map<String, Object> newProperties) {
-        parsedData.putAll(newProperties);
-        if (parsedData.size() > 0) {
-            parsedData.values().removeAll(Collections.singleton(null));
-            parsedData.values().removeAll(Collections.singleton(""));
-        }
-        return parsedData;
-    }
-
-    public Map<String, Object> extractProperties(Map<String, Object> message) {
-        Map<String, Object> properties = new HashMap<>();
-        Map transactionData = (Map) message.get("transactionData");
-        if (transactionData != null) {
-            Map<String, Object> addedProperties = (Map<String, Object>) transactionData.get("properties");
-            if (addedProperties != null && !addedProperties.isEmpty()) {
-                for (Map.Entry<String, Object> propertyMap : addedProperties.entrySet()) {
-                    if (propertyMap != null && propertyMap.getKey() != null && null!=((Map<String, Object>) propertyMap.getValue()).get("nv") && !((Map<String, Object>) propertyMap.getValue()).get("nv").toString().isEmpty()) {
-                        String propertyName = propertyMap.getKey();
-                        Object propertyNewValue = ((Map<String, Object>) propertyMap.getValue()).get("nv");
-                        properties.put(propertyName, propertyNewValue);
-                    }
-                }
-            }
-        }
-        return properties;
-    }
-
-    public Map<String, Object> convertType(Map<String, Object> newProperties, List<String> contentModelListTypeFields, List<String> dateFields) {
-        Map<String, Object> result = new HashMap();
-        for(String entry: contentModelListTypeFields){
-            if(newProperties.containsKey(entry)
-                    && newProperties.get(entry) instanceof String) {
-                String str = (String) newProperties.get(entry);
-                if(dateFields.contains(entry)) {
-                    try{
-                        Date date = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(str);
-                        result.put(entry,date.getTime());
-                    }catch (ParseException ex){ ex.printStackTrace();}
-                }
-                else {
-                    List<String> value = Arrays.asList(str);
-                    result.put(entry, value);
-                }
-            }
-        }
-        return result;
-    }
-
-    public Object getMetadata(String apiUrl, RestUtil restUtil, String authKey, String fieldName) {
-        Map<String, String> headers = new HashMap<>();
-        headers.put("Authorization", "Bearer " + authKey);
-        try {
-            String responseBody = restUtil.get(apiUrl, headers);
-            Gson gson = new Gson();
-            Map<String, Object> response = gson.fromJson(
-                    responseBody, new TypeToken<HashMap<String, Object>>() {
-                    }.getType()
-            );
-            return gson.fromJson(gson.toJson(response.get("result")), Map.class).get(fieldName);
-        } catch (NullPointerException e) {
-            LOGGER.error(null, "Exception while fetching metadata" + e);
-            e.printStackTrace();
-            return null;
-        }
-    }
-}
diff --git a/data-pipeline/content-cache-updater/src/main/resources/log4j.xml b/data-pipeline/content-cache-updater/src/main/resources/log4j.xml
deleted file mode 100644
index 7e0a688c9b..0000000000
--- a/data-pipeline/content-cache-updater/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-    <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-    <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-        <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-        <param name="append" value="true" />
-        <param name="maxFileSize" value="100MB" />
-        <param name="maxBackupIndex" value="5" />
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-        </layout>
-    </appender>
-    <root>
-        <priority value="info" />
-        <appender-ref ref="RollingAppender"/>
-        <appender-ref ref="jmx" />
-    </root>
-</log4j:configuration>
diff --git a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java b/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java
deleted file mode 100644
index 19fcc65a37..0000000000
--- a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/ContentCacheUpdaterServiceTest.java
+++ /dev/null
@@ -1,374 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import okhttp3.*;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.BaseCacheUpdaterService;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.Fixtures.EventFixture;
-import org.ekstep.ep.samza.task.ContentCacheConfig;
-import org.ekstep.ep.samza.task.ContentCacheUpdaterSink;
-import org.ekstep.ep.samza.task.ContentCacheUpdaterSource;
-import org.ekstep.ep.samza.task.ContentCacheUpdaterTask;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.util.RestUtil;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import redis.clients.jedis.Jedis;
-
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
-
-public class ContentCacheUpdaterServiceTest {
-
-    private RedisConnect redisConnectMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private JobMetrics jobMetricsMock;
-    private Jedis jedisMock = new MockJedis("test");
-    private ContentCacheUpdaterService contentCacheUpdaterService;
-    private ContentCacheUpdaterSink contentCacheUpdaterSinkMock;
-    private ContentCacheUpdaterTask contentCacheUpdaterTask;
-    private ContentCacheConfig contentCacheConfig;
-    private TaskContext contextMock;
-    private TaskCoordinator taskCoordinator;
-    private MessageCollector messageCollector;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private Config configMock;
-    private Integer contentStoreId = 5;
-    private Integer dialCodeStoreId = 6;
-    private static final String DIAL_CODE_API_ENDPOINT = "/api/dialcode/v3/read/";
-    private static final String DIAL_CODE_HOST = "https://localhost";
-    private static final String DIAL_CODE_API_KEY = "";
-    private RestUtil restUtilMock;
-
-
-    @Before
-    public void setUp() throws IOException {
-        redisConnectMock = mock(RedisConnect.class);
-        contentCacheUpdaterSinkMock = mock(ContentCacheUpdaterSink.class);
-        configMock = mock(Config.class);
-        contextMock = mock(TaskContext.class);
-        taskCoordinator = mock(TaskCoordinator.class);
-        messageCollector = mock(MessageCollector.class);
-        jobMetricsMock = mock(JobMetrics.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = mock(Counter.class);
-        restUtilMock = mock(RestUtil.class);
-
-        stub(redisConnectMock.getConnection(contentStoreId)).toReturn(jedisMock);
-        stub(redisConnectMock.getConnection(dialCodeStoreId)).toReturn(jedisMock);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        stub(configMock.getInt("redis.database.contentStore.id", contentStoreId)).toReturn(contentStoreId);
-        stub(configMock.getInt("redis.database.dialCodeStore.id", dialCodeStoreId)).toReturn(dialCodeStoreId);
-        stub(configMock.getInt("location.db.redis.key.expiry.seconds", 86400)).toReturn(86400);
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "learning.graph.events", new Partition(0)));
-        stub(configMock.get("dialcode.api.host", DIAL_CODE_HOST)).toReturn(DIAL_CODE_HOST);
-        stub(configMock.get("dialcode.api.endpoint", DIAL_CODE_API_ENDPOINT)).toReturn(DIAL_CODE_API_ENDPOINT);
-        stub(configMock.get("dialcode.api.authorizationkey", DIAL_CODE_API_KEY)).toReturn(DIAL_CODE_API_KEY);
-        List<String> defaultListValues = new ArrayList<>();
-        defaultListValues.add("gradeLevel");
-        defaultListValues.add("subject");
-        defaultListValues.add("medium");
-        defaultListValues.add("language");
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(configMock.getList("contentModel.fields.listType", new ArrayList<>()))
-                .toReturn(defaultListValues);
-        contentCacheConfig = new ContentCacheConfig(configMock);
-        new BaseCacheUpdaterService(redisConnectMock);
-        String validDialCodeUrl = "https://localhost/api/dialcode/v3/read/E1L8W5";
-        String inValidDialCodeUrl_Case1 = "https://localhost/api/dialcode/v3/read/test";
-        String inValidDialCodeUrl_Case2 = "https://localhost/api/dialcode/v3/read/4328W56";
-        String validDialCodeUrl_case1 = "https://localhost/api/dialcode/v3/read/Z5A8W1";
-        String validDialCodeUrl_case2 = "https://localhost/api/dialcode/v3/read/H4D5Q9";
-        String validDialCodeUrl_case3 = "https://localhost/api/dialcode/v3/read/C1M7J9";
-        createStub(validDialCodeUrl, createTestResponse(validDialCodeUrl, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        createStub(validDialCodeUrl_case1, createTestResponse(validDialCodeUrl_case1, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        createStub(validDialCodeUrl_case2, createTestResponse(validDialCodeUrl_case2, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        createStub(validDialCodeUrl_case3, createTestResponse(validDialCodeUrl_case3, EventFixture.VALID_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        createStub(inValidDialCodeUrl_Case1, createTestResponse(inValidDialCodeUrl_Case1, EventFixture.INVALID_DIAL_CODE_RESPONSE, 404, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        createStub(inValidDialCodeUrl_Case2, createTestResponse(inValidDialCodeUrl_Case2, EventFixture.EMPTY_DIAL_CODE_RESPONSE, 200, contentCacheConfig.getAuthorizationKey()), contentCacheConfig.getAuthorizationKey());
-        contentCacheUpdaterService = new ContentCacheUpdaterService(contentCacheConfig, redisConnectMock, jobMetricsMock, restUtilMock);
-        jedisMock.flushAll();
-    }
-
-
-    public String createTestResponse(String apiUrl, String response, int status, String authKey) throws IOException {
-        Request mockRequest = new Request.Builder()
-                .url(apiUrl)
-                .header("Authorization", authKey)
-                .build();
-        return Objects.requireNonNull(new Response.Builder()
-                .request(mockRequest)
-                .protocol(Protocol.HTTP_2)
-                .code(status) // status code
-                .message("")
-                .body(ResponseBody.create(
-                        MediaType.get("application/json; charset=utf-8"),
-                        response
-                ))
-                .build().body()).string();
-    }
-
-    public void createStub(String apiUrl, String response, String authKey) {
-        Map<String, String> headers = new HashMap<String, String>();
-        headers.put("Authorization", authKey);
-        try {
-            when(restUtilMock.get(apiUrl, headers)).thenReturn(response);
-        } catch (Exception e) {
-            System.out.println("Exception is" + e);
-        }
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void updateContentsToContentStoreCache() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1);
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1, Map.class);
-        String contentId = (String) event.get("nodeUniqueId");
-
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        Map<String, Object> contentData = contentCacheUpdaterService.getCacheData(source.getMap());
-        jedisMock.set(contentId, contentData.toString());
-
-        String cachedData = jedisMock.get(contentId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(5, parsedData.size());
-        assertEquals("in.ekstep", parsedData.get("channel"));
-        assertEquals("TestCollection", parsedData.get("description"));
-        assertEquals("testbook1", parsedData.get("code"));
-        assertEquals("Draft", parsedData.get("status"));
-        assertEquals(parsedData.get("ownershipType") instanceof List, true);
-        assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType"));
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void handleStringToList() throws Exception {
-        jedisMock.flushAll();
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INCORRECT_LIST_TYPE_FIELDS);
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.INCORRECT_LIST_TYPE_FIELDS, Map.class);
-        jedisMock.select(contentStoreId);
-        String contentId = (String) event.get("nodeUniqueId");
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        Map<String, Object> contentData = contentCacheUpdaterService.getCacheData(source.getMap());
-        jedisMock.set(contentId, contentData.toString());
-
-        String cachedData = jedisMock.get(contentId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(4, parsedData.size());
-        assertEquals(parsedData.get("language") instanceof List, true);
-        assertEquals(Arrays.asList("Spanish"), parsedData.get("language"));
-        assertEquals(parsedData.get("subject") instanceof List, true);
-        assertEquals(Arrays.asList("CS"), parsedData.get("subject"));
-        assertEquals(22.0, parsedData.get("ageGroup"));
-        assertEquals(parsedData.get("ownershipType") instanceof List, true);
-        assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType"));
-        verify(contentCacheUpdaterSinkMock, times(1)).success();
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void shouldUpdateCacheIfDataExist() throws Exception {
-        jedisMock.flushAll();
-
-        // Update Redis Cache with First Event
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1, EventFixture.OBJECT_TYPE_CONTENT_EVENT_1_UPDATED);
-        contentCacheUpdaterService.process(new ContentCacheUpdaterSource(envelopeMock), contentCacheUpdaterSinkMock);
-
-        // Update the redis cache with Second event with updated fields: channel, status
-        // removed fields: ownershipType
-        // new fields: visibility
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.OBJECT_TYPE_CONTENT_EVENT_1_UPDATED, Map.class);
-        String contentId = (String) event.get("nodeUniqueId");
-
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        Map<String, Object> contentData = contentCacheUpdaterService.getCacheData(source.getMap());
-        jedisMock.set(contentId, contentData.toString());
-        String cachedData = jedisMock.get(contentId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(6, parsedData.size());
-        assertEquals("testbook1", parsedData.get("code"));
-        assertEquals("sunbird.portal", parsedData.get("channel"));
-        assertEquals("Live", parsedData.get("status"));
-        assertEquals("Default", parsedData.get("visibility"));
-        assertEquals("TestCollection", parsedData.get("description"));
-        assertEquals(Arrays.asList("createdBy"), parsedData.get("ownershipType"));
-    }
-
-    @Test
-    public void shouldNotAddToCacheIfKeyValueIsNullOrEmpty() throws Exception {
-        jedisMock.flushAll();
-
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.CONTENT_EVENT_EMPTY_NODE_UNIQUEID, EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        assertEquals(0, jedisMock.keys("*").size());
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void shouldNotUpdateEmptyMessageProperties() throws Exception {
-        jedisMock.flushAll();
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-
-        // Update Redis Cache with First Event
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-
-        // Second event with empty properties
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.CONTENT_EVENT_EMPTY_PROPERTIES, Map.class);
-        String contentId = (String) event.get("nodeUniqueId");
-
-        String cachedData = jedisMock.get(contentId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(0, parsedData.size());
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void shouldAddOtherObjectTypeToCache() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.OBJECT_TYPE_CONCEPT_EVENT);
-
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.OBJECT_TYPE_CONCEPT_EVENT, Map.class);
-        String conceptId = (String) event.get("nodeUniqueId");
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        Map<String, Object> contentData = contentCacheUpdaterService.getCacheData(source.getMap());
-        jedisMock.set(conceptId, contentData.toString());
-
-        String cachedData = jedisMock.get(conceptId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(1, parsedData.size());
-        ArrayList<String> str = new ArrayList<>();
-        str.add("English");
-        assertEquals(str, parsedData.get("language"));
-    }
-
-    @Test
-    public void shouldMarkEventSkippedForNonodeUniqueId() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.CONTENT_EVENT_EMPTY_NODE_UNIQUEID);
-        contentCacheUpdaterTask = new ContentCacheUpdaterTask(configMock, contextMock, redisConnectMock, restUtilMock);
-        contentCacheUpdaterTask.process(envelopeMock, messageCollector, taskCoordinator);
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-
-        verify(contentCacheUpdaterSinkMock, times(1)).markSkipped();
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void shouldNotAddEventIfItIsEmpty() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_LANGUAGE_FIELD_EVENT);
-
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.EMPTY_LANGUAGE_FIELD_EVENT, Map.class);
-        String conceptId = (String) event.get("nodeUniqueId");
-
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-        Map<String, Object> contentData = contentCacheUpdaterService.getCacheData(source.getMap());
-        jedisMock.set(conceptId, contentData.toString());
-
-        String cachedData = jedisMock.get(conceptId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(0, parsedData.size());
-        assertEquals(null, parsedData.get("language"));
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void shouldInvokeDialCodeAPICall() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DIALCODE_LINKED_TEXTBOOK);
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.DIALCODE_LINKED_TEXTBOOK, Map.class);
-        String conceptId = (String) event.get("nodeUniqueId");
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void WhenDialCodeIsNotPresetInTheExternalSystemAPIShouldReturnResourceNotFound() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_DIALCODE_LINKED_TEXTBOOK);
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.INVALID_DIALCODE_LINKED_TEXTBOOK, Map.class);
-        String conceptId = (String) event.get("nodeUniqueId");
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-    }
-
-    @Test
-    @SuppressWarnings("unchecked")
-    public void WhenEmptyDialCodeAPIResponse() throws Exception {
-        jedisMock.flushAll();
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_DIALCODE_LINKED_TEXTBOOK);
-        Gson gson = new Gson();
-        Map<String, Object> event = gson.fromJson(EventFixture.EMPTY_DIALCODE_LINKED_TEXTBOOK, Map.class);
-        String conceptId = (String) event.get("nodeUniqueId");
-        ContentCacheUpdaterSource source = new ContentCacheUpdaterSource(envelopeMock);
-        contentCacheUpdaterService.process(source, contentCacheUpdaterSinkMock);
-    }
-}
diff --git a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
deleted file mode 100644
index edf7caccfb..0000000000
--- a/data-pipeline/content-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package org.ekstep.ep.samza.service.Fixtures;
-
-public class EventFixture {
-    public static final String OBJECT_TYPE_CONTENT_EVENT_1 = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"in.ekstep\"},\"description\":{\"ov\":null,\"nv\":\"TestCollection\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}";
-    public static final String OBJECT_TYPE_CONTENT_EVENT_1_UPDATED = "{\"ets\":1548780816826,\"channel\":\"sunbird.portal\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":null},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"sunbird.portal\"},\"status\":{\"ov\":null,\"nv\":\"Live\"},\"visibility\":{\"ov\":null,\"nv\":\"Default\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}";
-    public static final String OBJECT_TYPE_DIAL_CODE_1 = "{\"nodeUniqueId\":\"YC9EP8\",\"ets\":1549345256766,\"requestId\":null,\"audit\":false,\"transactionData\":{\"properties\":{\"dialcode_index\":{\"ov\":null,\"nv\":261351.0},\"identifier\":{\"ov\":null,\"nv\":\"YC9EP8\"},\"channel\":{\"ov\":null,\"nv\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"batchcode\":{\"ov\":null,\"nv\":\"do_112692236142379008136\"},\"publisher\":{\"ov\":null,\"nv\":null},\"generated_on\":{\"ov\":null,\"nv\":\"2019-02-05T05:40:56.762\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"channel\":\"in.ekstep\",\"index\":true,\"operationType\":\"CREATE\",\"nodeType\":\"EXTERNAL\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-02-05T05:40:56.766+0000\",\"objectType\":\"DialCode\"}";
-    public static final String CONTENT_EVENT_EMPTY_PROPERTIES = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"do_11268761245100441611\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}";
-    public static final String CONTENT_EVENT_EMPTY_NODE_UNIQUEID = "{\"ets\":1548780816826,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]},\"code\":{\"ov\":null,\"nv\":\"testbook1\"},\"channel\":{\"ov\":null,\"nv\":\"in.ekstep\"},\"description\":{\"ov\":null,\"nv\":\"TestCollection\"},\"status\":{\"ov\":null,\"nv\":\"Draft\"}}},\"label\":\"TestBook1\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-29T22:23:36.826+0530\",\"objectType\":\"Content\",\"nodeUniqueId\":\"\",\"requestId\":null,\"operationType\":\"CREATE\",\"nodeGraphId\":703654,\"graphId\":\"domain\"}";
-    public static final String OBJECT_TYPE_CONCEPT_EVENT = "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"English\"}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}";
-    public static final String EMPTY_LANGUAGE_FIELD_EVENT= "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"\"}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}";
-    public static final String LIST_LANGUAGE_FIELD_EVENT= "{\"ets\":1547644874028,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":[\"Kannada\"]}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-01-16T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":101409,\"graphId\":\"domain\"}";
-    public static final String INCORRECT_LIST_TYPE_FIELDS= "{\"ets\":1559670759,\"channel\":\"in.ekstep\",\"transactionData\":{\"properties\":{\"language\":{\"ov\":null,\"nv\":\"Spanish\"},\"subject\":{\"ov\":null,\"nv\":\"CS\"},\"ageGroup\":{\"ov\":null,\"nv\":\"22\"},\"ownershipType\":{\"ov\":null,\"nv\":[\"createdBy\"]}}},\"label\":\"Properties of Light\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-06-07T18:51:14.028+0530\",\"objectType\":\"Concept\",\"nodeUniqueId\":\"SMC4\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":98765,\"graphId\":\"domain\"}";
-    public static final String DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"E1L8W5\"]},\"reservedDialcodes\":{\"ov\":null,\"nv\":\"{\\\"Z5A8W1\\\":2,\\\"H4D5Q9\\\":0,\\\"C1M7J9\\\":1}\"},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}";
-    public static final String INVALID_DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"test\"]},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}";
-    public static final String VALID_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-03T09:09:50ZZ\",\"params\":{\"resmsgid\":\"fef4be15-b87e-421e-9e31-077bab44f0f2\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":{\"dialcode\":{\"identifier\":\"977D3I\",\"channel\":\"0123221617357783046602\",\"publisher\":\"MHPUBLISHER\",\"batchCode\":\"MHPUBLISHER.20180402T112421\",\"status\":\"Draft\",\"generatedOn\":\"2018-04-02T11:24:22.366\",\"publishedOn\":null,\"metadata\":null}}}";
-    public static final String EMPTY_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-03T09:09:50ZZ\",\"params\":{\"resmsgid\":\"fef4be15-b87e-421e-9e31-077bab44f0f2\",\"msgid\":null,\"err\":null,\"status\":\"successful\",\"errmsg\":null},\"responseCode\":\"OK\",\"result\":null}";
-    public static final String INVALID_DIAL_CODE_RESPONSE = "{\"id\":\"sunbird.dialcode.read\",\"ver\":\"3.0\",\"ts\":\"2020-03-04T10:00:18ZZ\",\"params\":{\"resmsgid\":\"df4a320b-2cc9-4bc8-9273-dc4f82ee4b43\",\"msgid\":null,\"err\":\"ERR_DIALCODE_INFO\",\"status\":\"failed\",\"errmsg\":\"Dial Code Not Found With Id: S6B4X11\"},\"responseCode\":\"RESOURCE_NOT_FOUND\",\"result\":{}}";
-    public static final String EMPTY_DIALCODE_LINKED_TEXTBOOK="{\"ets\":1577734412619,\"channel\":\"channel-01\",\"transactionData\":{\"properties\":{\"dialcodes\":{\"ov\":null,\"nv\":[\"4328W56\",\"\"]},\"SYS_INTERNAL_LAST_UPDATED_ON\":{\"ov\":\"2019-12-30T19:33:32.504+0000\",\"nv\":\"2019-12-30T19:33:32.580+0000\"}}},\"mid\":\"3c99290d-037e-47c6-98f8-ddbd26452fb8\",\"label\":\"KP Integration Test Collection Content\",\"nodeType\":\"DATA_NODE\",\"userId\":\"ANONYMOUS\",\"createdOn\":\"2019-12-30T19:33:32.619+0000\",\"objectType\":\"Content\",\"nodeUniqueId\":\"KP_FT_1577734411271\",\"requestId\":null,\"operationType\":\"UPDATE\",\"nodeGraphId\":373563,\"graphId\":\"domain\"}";
-}
diff --git a/data-pipeline/de-duplication/pom.xml b/data-pipeline/de-duplication/pom.xml
deleted file mode 100644
index 61028dc4a9..0000000000
--- a/data-pipeline/de-duplication/pom.xml
+++ /dev/null
@@ -1,179 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>de-duplication</artifactId>
-    <version>0.5.1</version>
-    <packaging>jar</packaging>
-    <name>Deduplication</name>
-    <description>
-       Identifying Duplicate Events
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/de-duplication/src/main/assembly/src.xml b/data-pipeline/de-duplication/src/main/assembly/src.xml
deleted file mode 100644
index 437104ac07..0000000000
--- a/data-pipeline/de-duplication/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/de-duplication.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:de-duplication</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/de-duplication/src/main/config/de-duplication.properties b/data-pipeline/de-duplication/src/main/config/de-duplication.properties
deleted file mode 100644
index 436ef9e702..0000000000
--- a/data-pipeline/de-duplication/src/main/config/de-duplication.properties
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.DeDuplication
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_de_duplication_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.DeDuplicationTask
-task.inputs=kafka.__env__.telemetry.valid
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# YARN container configuration
-# yarn.am.container.memory.mb=__yarn_am_container_mb__
-# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__
-
-output.success.topic.name=__env__.telemetry.unique
-output.failed.topic.name=__env__.telemetry.failed
-output.duplicate.topic.name=__env__.telemetry.duplicate
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,duplicate-event-count
-default.channel=__default_channel__
-
-
-# redis
-#redis.host=localhost
-redis.host=__redis_host__
-#redis.port=6379
-redis.port=__redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.duplicationstore.id=7
-redis.database.key.expiry.seconds=86400
-
-
-dedup.producer.include.ids=__dedup_include_producer_ids__
-
-
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 0d40545d34..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-
-import org.apache.commons.lang.StringUtils;
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    public Event(Map<String, Object> map) {
-
-        super(map);
-    }
-
-    public void markSkipped() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_processed", false);
-        telemetry.add("flags.dd_checksum_present", false);
-    }
-
-    public void markDuplicate() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_processed", false);
-        telemetry.add("flags.dd_duplicate_event", true);
-    }
-
-    public void markSuccess() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_processed", true);
-        telemetry.add("type", "events");
-    }
-
-    public void markRedisFailure() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_processed", false);
-        telemetry.add("flags.dd_redis_failure", true);
-        telemetry.add("type", "events");
-    }
-
-    public void updateDefaults(DeDuplicationConfig config) {
-        String channelString = telemetry.<String>read("context.channel").value();
-        String channel = StringUtils.deleteWhitespace(channelString);
-        if (channel == null || channel.isEmpty()) {
-            telemetry.addFieldIfAbsent("context", new HashMap<String, Object>());
-            telemetry.add("context.channel", config.defaultChannel());
-        }
-    }
-}
-
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java
deleted file mode 100644
index 1cc36c771a..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import org.ekstep.ep.samza.task.DeDuplicationSink;
-import org.ekstep.ep.samza.task.DeDuplicationSource;
-import org.ekstep.ep.samza.util.DeDupEngine;
-
-import redis.clients.jedis.exceptions.JedisException;
-
-public class DeDuplicationService {
-  private static Logger LOGGER = new Logger(DeDuplicationService.class);
-  private final DeDupEngine deDupEngine;
-  private final DeDuplicationConfig config;
-
-  public DeDuplicationService(DeDupEngine deDupEngine, DeDuplicationConfig config) {
-    this.deDupEngine = deDupEngine;
-    this.config = config;
-  }
-
-  public void process(DeDuplicationSource source, DeDuplicationSink sink) throws Exception {
-    Event event = null;
-
-    try {
-      event = source.getEvent();
-      String checksum = event.getChecksum();
-      if (checksum == null) {
-        LOGGER.info(event.id(), "EVENT WITHOUT CHECKSUM & MID, PASSING THROUGH : {}", event);
-        event.markSkipped();
-        event.updateDefaults(config);
-        sink.toSuccessTopic(event);
-        return;
-      }
-
-      if (isDupCheckRequired(event)) {
-        if (!deDupEngine.isUniqueEvent(checksum)) {
-          LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum);
-          event.markDuplicate();
-          sink.toDuplicateTopic(event);
-          return;
-        }
-
-        LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE");
-
-        deDupEngine.storeChecksum(checksum);
-      }
-      event.updateDefaults(config);
-      event.markSuccess();
-      sink.toSuccessTopic(event);
-
-    } catch (JedisException e) {
-      e.printStackTrace();
-      LOGGER.error(null, "Exception when retrieving data from redis: ", e);
-      deDupEngine.getRedisConnection().close();
-      throw e;
-    } catch (JsonSyntaxException e) {
-      LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-      sink.toMalformedEventsTopic(source.getMessage());
-    }
-  }
-
-  public boolean isDupCheckRequired(Event event) {
-    return (config.inclusiveProducerIds().isEmpty()
-        || (null != event.producerId() && config.inclusiveProducerIds().contains(event.producerId())));
-  }
-}
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java
deleted file mode 100644
index efca4a5e40..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DeDuplicationConfig {
-
-    private final String JOB_NAME = "DeDuplication";
-    private String successTopic;
-    private String failedTopic;
-    private String duplicateTopic;
-    private String malformedTopic;
-    private String defaultChannel;
-    private final int dupStore;
-    private int expirySeconds;
-    private List<String> includedProducerIds = new ArrayList<>();
-
-
-    public DeDuplicationConfig(Config config) {
-        successTopic = config.get("output.success.topic.name", "telemetry.valid");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        defaultChannel = config.get("default.channel", "org.sunbird");
-        dupStore = config.getInt("redis.database.duplicationstore.id", 7);
-        expirySeconds = config.getInt("redis.database.key.expiry.seconds", 86400);
-        if (!config.get("dedup.producer.include.ids", "").isEmpty()) {
-            includedProducerIds = config.getList("dedup.producer.include.ids", new ArrayList<>());
-        }
-
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String duplicateTopic() {
-        return duplicateTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String defaultChannel() {
-        return defaultChannel;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public int dupStore() {
-        return dupStore;
-    }
-
-    public int expirySeconds() {
-        return expirySeconds;
-    }
-
-    public List<String> inclusiveProducerIds() {
-        return includedProducerIds;
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java
deleted file mode 100644
index a100e4be50..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class DeDuplicationSink extends BaseSink {
-
-    private DeDuplicationConfig config;
-
-    public DeDuplicationSink(MessageCollector collector, JobMetrics metrics, DeDuplicationConfig config) {
-        super(collector, metrics);
-        this.config = config;
-    }
-
-    public void toSuccessTopic(Event event) {
-        toTopic(config.successTopic(), event.did(), event.getJson());
-        metrics.incSuccessCounter();
-    }
-
-    public void toSuccessTopicIfRedisException(Event event) {
-        toTopic(config.successTopic(), event.did(), event.getJson());
-        metrics.incCacheErrorCounter();
-    }
-
-    public void toDuplicateTopic(Event event) {
-        toTopic(config.duplicateTopic(), event.did(), event.getJson());
-        metrics.incDuplicateCounter();
-    }
-
-    public void toMalformedEventsTopic(String message) {
-        toTopic(config.malformedTopic(), null, message);
-        metrics.incFailedCounter();
-    }
-
-    public void toErrorTopic(Event event) {
-        toTopic(config.failedTopic(), event.did(), event.getJson());
-        metrics.incErrorCounter();
-    }
-
-}
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java
deleted file mode 100644
index 533787e194..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class DeDuplicationSource {
-	static Logger LOGGER = new Logger(DeDuplicationSource.class);
-
-	private IncomingMessageEnvelope envelope;
-
-	public DeDuplicationSource(IncomingMessageEnvelope envelope) {
-		this.envelope = envelope;
-	}
-
-	public Event getEvent() {
-		String message = (String) envelope.getMessage();
-		@SuppressWarnings("unchecked")
-		Map<String, Object> jsonMap = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-		return new Event(jsonMap);
-	}
-
-	public String getMessage() {
-		return envelope.toString();
-	}
-
-}
diff --git a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java b/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java
deleted file mode 100644
index b168b19292..0000000000
--- a/data-pipeline/de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.DeDuplicationService;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class DeDuplicationTask extends BaseSamzaTask {
-
-	private DeDuplicationConfig config;
-	private JobMetrics metrics;
-	private DeDuplicationService service;
-
-	public DeDuplicationTask(Config config, TaskContext context,
-							 DeDupEngine deDupEngine) {
-		init(config, context, deDupEngine);
-	}
-
-	public DeDuplicationTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		init(config, context, null);
-	}
-
-	private void init(Config config, TaskContext context,
-					  DeDupEngine deDupEngine) {
-		this.config = new DeDuplicationConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		deDupEngine = deDupEngine == null ?
-				new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine;
-		service = new DeDuplicationService(deDupEngine, this.config);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
-		DeDuplicationSource source = new DeDuplicationSource(envelope);
-		DeDuplicationSink sink = new DeDuplicationSink(collector, metrics, config);
-		service.process(source, sink);
-	}
-
-}
diff --git a/data-pipeline/de-duplication/src/main/resources/log4j.xml b/data-pipeline/de-duplication/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/de-duplication/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
deleted file mode 100644
index 82294daabe..0000000000
--- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Map;
-
-public class EventTest {
-
-    @Test
-    public void shouldMarkTheRedisFailure() {
-        String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-        Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-        event.markRedisFailure();
-        Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-        Object type = event.getMap().get("type");
-        Assert.assertEquals(type, "events");
-        Assert.assertNotNull(flagData);
-        Assert.assertEquals(flagData.get("dd_processed"), false);
-        Assert.assertEquals(flagData.get("dd_redis_failure"), true);
-        Assert.assertNotNull(type);
-
-    }
-
-    @Test
-    public void shouldMarkSkipped() {
-        String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-        Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-        event.markSkipped();
-        Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-        Assert.assertNotNull(flagData);
-        Assert.assertEquals(flagData.get("dd_processed"), false);
-        Assert.assertEquals(flagData.get("dd_checksum_present"), false);
-
-    }
-
-
-}
diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index ce61e49754..0000000000
--- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,166 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.util.Map;
-
-public class EventFixture {
-    private static final String EVENT_WITH_CHECKSUM = "{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"9a78dacddfcdfae7a9679b143e348158\"\n" +
-            "  },\n" +
-            "  \"eid\": \"INTERACT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"id\": \"do_312764599773110272147\",\n" +
-            "    \"type\": \"TOUCH\",\n" +
-            "    \"pageid\": \"ContentApp-EndScreen\",\n" +
-            "    \"subtype\": \"ContentID\"\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561620619274,\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"ver\": \"1.14.5\",\n" +
-            "      \"pid\": \"sunbird-portal.contentplayer\",\n" +
-            "      \"id\": \"dev.diksha.portal\"\n" +
-            "    },\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"env\": \"contentplayer\",\n" +
-            "    \"did\": \"9a78dacddfcdfae7a9679b143e348158\",\n" +
-            "    \"sid\": \"8c416b30-4e47-a695-b1d2-9cd030eadeff\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"DialCode\",\n" +
-            "        \"id\": \"DNILBR\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"ContentSession\",\n" +
-            "        \"id\": \"88de9e1dfa6f237c076278a02cc4b305\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"INTERACT:fd63ff6bd0eedf05cf57c0058588ae69\",\n" +
-            "  \"object\": {\n" +
-            "    \"ver\": \"1\",\n" +
-            "    \"id\": \"do_312764599773110272147\",\n" +
-            "    \"type\": \"Content\",\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"do_31276594374681395211909\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"syncts\": 1561620619795,\n" +
-            "  \"@timestamp\": \"2019-06-27T07:30:19.795Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    private static final String EVENT_WITH_MID = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\",\n" +
-            "    \"mid\": \"22e1430f2e5f339230dbf9595b060008\"" +
-            "}";
-
-    private static final String EVENT_WITHOUT_CHECKSUM = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\"" +
-            "}";
-
-    private static final String EVENT_WITH_EMTPY_CHANNEL = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"context\": {\n" +
-            "    \"channel\": \"\"\n" +
-            "    },\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\"" +
-            "}";
-
-    public static Map<String, Object> EventWithChecksumMap() {
-        return new Gson().fromJson(EVENT_WITH_CHECKSUM, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithMidMap() {
-        return new Gson().fromJson(EVENT_WITH_MID, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithoutChecksumFieldMap() {
-        return new Gson().fromJson(EVENT_WITHOUT_CHECKSUM, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithEmptyChannel() {
-        return new Gson().fromJson(EVENT_WITH_EMTPY_CHANNEL, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static String EventWithChecksumJson() {
-        return EVENT_WITH_CHECKSUM;
-    }
-
-}
-
diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java
deleted file mode 100644
index d29ab1f347..0000000000
--- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import org.junit.Assert;
-import org.junit.Test;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class EventTest {
-    @Test
-    public void shouldReturnChecksumIfPresent(){
-
-        Event event = new Event(EventFixture.EventWithChecksumMap());
-        Assert.assertEquals("INTERACT:fd63ff6bd0eedf05cf57c0058588ae69", (String) event.getChecksum());
-    }
-
-    @Test
-    public void shouldReturnMidIfPresent(){
-
-        Event event = new Event(EventFixture.EventWithMidMap());
-        Assert.assertEquals("22e1430f2e5f339230dbf9595b060008", (String) event.getChecksum());
-    }
-
-    @Test
-    public void shouldReturnNullIfChecksumAndMidAreAbsent(){
-
-        Event event = new Event(EventFixture.EventWithoutChecksumFieldMap());
-        Assert.assertEquals(null, (String) event.getChecksum());
-    }
-
-    @Test
-    public void shouldAddChannelIfChannelIsAbsentNullOrEmpty(){
-        DeDuplicationConfig conf = mock(DeDuplicationConfig.class);
-        when(conf.defaultChannel()).thenReturn("in.ekstep");
-
-        Event event = new Event(EventFixture.EventWithMidMap());
-        event.updateDefaults(conf);
-        Assert.assertEquals("in.ekstep", ( (Map<String,Object>) event.getMap().get("context")).get("channel") );
-
-
-        Event event2 = new Event(EventFixture.EventWithEmptyChannel());
-        event2.updateDefaults(conf);
-        Assert.assertEquals("in.ekstep", ( (Map<String,Object>) event.getMap().get("context")).get("channel"));
-    }
-}
-
diff --git a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java b/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java
deleted file mode 100644
index 20e842eaab..0000000000
--- a/data-pipeline/de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-import redis.clients.jedis.Jedis;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class DeDuplicationTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.valid";
-    private static final String FAILED_TOPIC = "telemetry.unique.fail";
-    private static final String DUPLICATE_TOPIC = "telemetry.duplicate";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private DeDuplicationTask deDuplicationTask;
-    private DeDupEngine deDupEngineMock;
-    private Jedis jedisMock = new MockJedis("duplicationtest");
-    private int dupStoreId = 1;
-
-    @Before
-    public void setUp() {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = Mockito.mock(TaskContext.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = Mockito.mock(Config.class);
-        RedisConnect redisConnectMock = mock(RedisConnect.class);
-        deDupEngineMock = mock(DeDupEngine.class);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC);
-        stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-        stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC);
-        stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-        stub(metricsRegistry.newCounter(anyString(), anyString()))
-                .toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(0)));
-        stub(configMock.getInt("redis.database.duplicationstore.id", dupStoreId)).toReturn(dupStoreId);
-        when(configMock.get("dedup.producer.include.ids", "")).thenReturn("dev.diskha.portal");
-        stub(configMock.getList("dedup.producer.include.ids", new ArrayList<>())).toReturn(Arrays.asList("dev.diksha.portal"));
-
-
-        deDuplicationTask = new DeDuplicationTask(configMock, contextMock, deDupEngineMock);
-    }
-
-    @Test
-    public void ShouldSendEventToSuccessTopicIfEventIsUnique() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson());
-
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    @Test
-    public void ShouldSendEventsToDuplicateTopicIfEventIsDuplicate() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC)));
-    }
-
-    @Test
-    public void ShouldSendEventsToMalformedTopicIfEventIsMalformed() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn("{'metadata':{'checksum':'sajksajska'}");
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-    }
-
-    @Test(expected = Exception.class)
-    public void ShouldSendEventsToFailedTopicForAnyUnhandledException() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenThrow(new Exception());
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    public void ShouldStoreChecksumIfEventIsUnique() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(deDupEngineMock, times(1)).isUniqueEvent(anyString());
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-}
diff --git a/data-pipeline/de-normalization/pom.xml b/data-pipeline/de-normalization/pom.xml
deleted file mode 100644
index 201f8201c3..0000000000
--- a/data-pipeline/de-normalization/pom.xml
+++ /dev/null
@@ -1,188 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>de-normalization</artifactId>
-    <version>0.1.7</version>
-    <packaging>jar</packaging>
-    <name>De-normalizer</name>
-    <description>
-       Denormalise device, user and content data
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/de-normalization/src/main/assembly/src.xml b/data-pipeline/de-normalization/src/main/assembly/src.xml
deleted file mode 100644
index c01210c393..0000000000
--- a/data-pipeline/de-normalization/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/de-normalization.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:de-normalization</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/de-normalization/src/main/config/de-normalization.properties b/data-pipeline/de-normalization/src/main/config/de-normalization.properties
deleted file mode 100644
index b5e995aebd..0000000000
--- a/data-pipeline/de-normalization/src/main/config/de-normalization.properties
+++ /dev/null
@@ -1,101 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.DeNormalization
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__denormalization_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.DeNormalizationTask
-task.inputs=kafka.__env__.telemetry.with_location
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# YARN container configuration
-# yarn.am.container.memory.mb=__yarn_am_container_mb__
-# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__
-
-output.success.topic.name=__env__.telemetry.denorm
-output.failed.topic.name=__env__.telemetry.failed
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,failed-message-count,skipped-message-count,error-message-count,cache-hit-count,cache-empty-values-count,user-cache-hit-count,expired-event-count
-
-# redis
-# redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-location.db.redis.key.expiry.seconds=__location_db_redis_key_expiry_seconds__
-
-# redis database index
-redis.userDB.index=4
-redis.contentDB.index=5
-redis.dialcodeDB.index=6
-
-# metadata fields
-content.metadata.fields=__content_metadata_fields__
-user.metadata.fields=__user_metadata_fields__
-dialcode.metadata.fields=__dialcode_metadata_fields__
-
-user.signin.type.default = __user_sigin_type_default__
-user.login.type.defalut = __user_login_type_default__
-
-telemetry.ignore.period.months=6
-summary.filter.events=ME_WORKFLOW_SUMMARY
\ No newline at end of file
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java
deleted file mode 100644
index b0b7fe0fd2..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/AbstractFactory.java
+++ /dev/null
@@ -1,6 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-public interface AbstractFactory<T> {
-    public T getInstance(String type);
-}
-
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java
deleted file mode 100644
index 4568aadd0b..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/CollectionDataUpdater.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.util.ContentDataCache;
-
-public class CollectionDataUpdater extends IEventUpdater {
-
-    CollectionDataUpdater(ContentDataCache contentCache) {
-        this.dataCache = contentCache;
-        this.cacheType = "collection";
-    }
-
-    public void update(Event event){
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java
deleted file mode 100644
index dde475a52a..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/ContentDataUpdater.java
+++ /dev/null
@@ -1,15 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.util.ContentDataCache;
-
-public class ContentDataUpdater extends IEventUpdater {
-
-    ContentDataUpdater(ContentDataCache contentCache) {
-        this.dataCache = contentCache;
-        this.cacheType = "content";
-    }
-
-    public void update(Event event)  {
-    }
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java
deleted file mode 100644
index 52f9c30c86..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/DialcodeDataUpdater.java
+++ /dev/null
@@ -1,14 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.util.DialCodeDataCache;
-
-public class DialcodeDataUpdater extends IEventUpdater {
-
-    DialcodeDataUpdater(DialCodeDataCache dialcodeCache) {
-        this.dataCache = dialcodeCache;
-        this.cacheType = "dialcode";
-    }
-
-    public void update(Event event) {
-    }
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 9b4d4e415a..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,191 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.task.DeNormalizationConfig;
-import org.joda.time.DateTime;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.math.BigDecimal;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
-import java.util.*;
-
-public class Event extends Events {
-    private DateTimeFormatter dateFormat = DateTimeFormat.forPattern("yyyy-MM-dd").withZoneUTC();
-    private DateFormat formatter = new SimpleDateFormat("yyyy-MM-dd");
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public Long getEndTimestampOfDay(String date) {
-        Long ets = dateFormat.parseDateTime(date).plusHours(23).plusMinutes(59).plusSeconds(59).getMillis();
-        return ets;
-    }
-
-    public Long compareAndAlterEts() {
-        Long eventEts = ets();
-        Long currentMillis = new Date().getTime();
-        String currentDate = formatter.format(currentMillis);
-        Long endTsOfCurrentDate = getEndTimestampOfDay(currentDate);
-        if (eventEts > endTsOfCurrentDate) telemetry.add("ets", currentMillis);
-        return ets();
-    }
-
-    public Boolean isOlder(Integer periodInMonths) {
-        Long eventEts = ets();
-        Long periodInMillis = new DateTime().minusMonths(periodInMonths).getMillis();
-        if (eventEts < periodInMillis) return true;
-        else return false;
-    }
-
-
-    public String objectRollUpl1ID() {
-        if (objectFieldsPresent() && objectRollUpl1FieldsPresent()) {
-            return telemetry.<String>read("object.rollup.l1").value();
-        } else return null;
-    }
-
-
-    public boolean objectRollUpl1FieldsPresent() {
-        if (objectFieldsPresent()) {
-            String objectrollUpl1 = telemetry.<String>read("object.rollup.l1").value();
-            return null != objectrollUpl1 && !objectrollUpl1.isEmpty();
-        } else return false;
-    }
-
-    public boolean checkObjectIdNotEqualsRollUpl1Id() {
-        if (objectRollUpl1FieldsPresent() && !(objectID().equals(objectRollUpl1ID())))
-            return true;
-        else
-            return false;
-    }
-
-    public String getKey(String type) {
-        switch (type) {
-            case "device":
-                return did();
-            case "user":
-                if ("system".equalsIgnoreCase(actorType())) {
-                    return "";
-                } else {
-                    return actorId();
-                }
-            case "content":
-                return objectID();
-            case "collection":
-                return objectRollUpl1ID();
-            default:
-                return "";
-        }
-    }
-
-    public void addMetaData(String type, Map newData) {
-        switch (type) {
-            case "device":
-                addDeviceData(newData);
-                break;
-            case "user":
-                addUserData(newData);
-                break;
-            case "content":
-                addContentData(newData);
-                break;
-            case "dialcode":
-                addDialCodeData(newData);
-                break;
-            case "collection":
-                addCollectionData(newData);
-            default:
-                break;
-
-        }
-    }
-
-    public Map addISOStateCodeToDeviceData(Map deviceData) {
-        // add new statecode field
-        String statecode = deviceData.get("statecode").toString();
-        if (statecode != null && !statecode.isEmpty()) {
-            String iso3166statecode = "IN-" + statecode;
-            deviceData.put("iso3166statecode", iso3166statecode);
-            return deviceData;
-        } else return deviceData;
-    }
-
-    public void addDeviceData(Map newData) {
-        NullableValue<Map<String, Object>> previousData = telemetry.read(path.deviceData());
-        Map<String, Object> deviceData = previousData.isNull() ? new HashMap<>() : previousData.value();
-        // add new statecode field
-        if (!deviceData.isEmpty()) {
-            deviceData = addISOStateCodeToDeviceData(deviceData);
-        }
-        deviceData.putAll(newData);
-        telemetry.add(path.deviceData(), deviceData);
-        setFlag(DeNormalizationConfig.getDeviceLocationJobFlag(), true);
-    }
-
-    public void addUserData(Map newData) {
-        NullableValue<Map<String, Object>> previousData = telemetry.read(path.userData());
-        Map<String, Object> userdata = previousData.isNull() ? new HashMap<>() : previousData.value();
-        userdata.putAll(newData);
-        telemetry.add(path.userData(), userdata);
-        if (newData.size() > 2)
-            setFlag(DeNormalizationConfig.getUserLocationJobFlag(), true);
-        else
-            setFlag(DeNormalizationConfig.getUserLocationJobFlag(), false);
-    }
-
-    public void addContentData(Map newData) {
-        NullableValue<Map<String, Object>> previousData = telemetry.read(path.contentData());
-        Map<String, Object> contentData = previousData.isNull() ? new HashMap<>() : previousData.value();
-        contentData.putAll(newData);
-        telemetry.add(path.contentData(), contentData);
-        setFlag(DeNormalizationConfig.getContentLocationJobFlag(), true);
-    }
-
-    public void addCollectionData(Map newData) {
-        NullableValue<Map<String, Object>> previousData = telemetry.read(path.collectionData());
-        Map<String, Object> collectionData = previousData.isNull() ? new HashMap<>() : previousData.value();
-        collectionData.putAll(newData);
-        telemetry.add(path.collectionData(), collectionData);
-        setFlag(DeNormalizationConfig.getCollectionLocationJobFlag(), true);
-    }
-
-    public void addDialCodeData(Map newData) {
-        telemetry.add(path.dialCodeData(), newData);
-        setFlag(DeNormalizationConfig.getDialCodeLocationJobFlag(), true);
-    }
-
-    public void setFlag(String key, Object value) {
-        NullableValue<Map<String, Object>> telemetryFlag = telemetry.read(path.flags());
-        Map<String, Object> flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value();
-        if (!key.isEmpty()) flags.put(key, value);
-        telemetry.add(path.flags(), flags);
-    }
-
-    public Map<String, Object> getFlag() {
-        NullableValue<Map<String, Object>> telemetryFlag = telemetry.read(path.flags());
-        Map<String, Object> flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value();
-        return flags;
-    }
-
-    public String getUpgradedVersion() {
-        BigDecimal updatedVer = BigDecimal.valueOf(Double.parseDouble(telemetry.read(path.ver()).value().toString())).add(BigDecimal.valueOf(0.1));
-        return updatedVer.toString();
-    }
-
-    public void markFailure(String error, DeNormalizationConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.denorm_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("metadata.denorm_error", error);
-        telemetry.add("metadata.src", config.jobName());
-    }
-
-
-}
-
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java
deleted file mode 100644
index 300e3311c5..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/EventUpdaterFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.util.ContentDataCache;
-import org.ekstep.ep.samza.util.DialCodeDataCache;
-import org.ekstep.ep.samza.util.UserDataCache;
-
-
-public class EventUpdaterFactory implements AbstractFactory {
-
-    private ContentDataUpdater contentDataUpdater;
-    private UserDataUpdater userDataUpdater;
-    private DialcodeDataUpdater dialCodeDataUpdater;
-    private CollectionDataUpdater collectionDataUpdater;
-
-    public EventUpdaterFactory(ContentDataCache contentDataCache,
-                               UserDataCache userDataCache,
-                               DialCodeDataCache dialCodeDataCache) {
-
-        this.contentDataUpdater = new ContentDataUpdater(contentDataCache);
-        this.userDataUpdater = new UserDataUpdater(userDataCache);
-        this.dialCodeDataUpdater = new DialcodeDataUpdater(dialCodeDataCache);
-        this.collectionDataUpdater = new CollectionDataUpdater(contentDataCache);
-    }
-
-    public IEventUpdater getInstance(String type) {
-
-        switch (type) {
-            case "content-data-updater":
-                return this.contentDataUpdater;
-            case "user-data-updater":
-                return this.userDataUpdater;
-            case "dialcode-data-updater":
-                return this.dialCodeDataUpdater;
-            case "collection-data-updater":
-                return this.collectionDataUpdater;
-            default:
-                return null;
-        }
-    }
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java
deleted file mode 100644
index 26a7f7bb3c..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/IEventUpdater.java
+++ /dev/null
@@ -1,97 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.task.DeNormalizationConfig;
-import org.ekstep.ep.samza.util.DataCache;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.util.Map;
-
-public abstract class IEventUpdater {
-
-    static Logger LOGGER = new Logger(IEventUpdater.class);
-    DataCache dataCache;
-    String cacheType;
-    DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSZ").withZoneUTC();
-    DateTimeFormatter df1 = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").withZoneUTC();
-
-    public abstract void update(Event event);
-
-    public void update(Event event, String key) {
-
-        if (key != null && !key.isEmpty()) {
-            Map data = dataCache.getData(key);
-            if (data != null && !data.isEmpty()) {
-                event.addMetaData(cacheType, getConvertedData(data));
-            } else {
-                event.setFlag(DeNormalizationConfig.getJobFlag(cacheType), false);
-            }
-        }
-    }
-
-    private Map getConvertedData(Map data) {
-        if("content".equals(cacheType) || "collection".equals((cacheType)))
-            return getEpochConvertedContentDataMap(data);
-        else if("dialcode".equals(cacheType))
-            return getEpochConvertedDialcodeDataMap(data);
-        else
-            return data;
-
-    }
-
-    private Long getTimestamp(String ts, DateTimeFormatter df) {
-        try {
-            return df.parseDateTime(ts).getMillis();
-        } catch (Exception ex) {
-            return 0L;
-        }
-    }
-
-    private Long getConvertedTimestamp(String ts) {
-        Long epochTs = getTimestamp(ts, df);
-        if (epochTs == 0) {
-            epochTs = getTimestamp(ts, df1);
-        }
-        return epochTs;
-    }
-
-
-    private Map getEpochConvertedContentDataMap(Map data) {
-
-        Object lastSubmittedOn = data.get("lastsubmittedon");
-        Object lastUpdatedOn = data.get("lastupdatedon");
-        Object lastPublishedOn = data.get("lastpublishedon");
-        if (lastSubmittedOn instanceof String) {
-            lastSubmittedOn = getConvertedTimestamp(lastSubmittedOn.toString());
-        }
-        if (lastUpdatedOn instanceof String) {
-            lastUpdatedOn = getConvertedTimestamp(lastUpdatedOn.toString());
-        }
-        if (lastPublishedOn instanceof String) {
-            lastPublishedOn = getConvertedTimestamp(lastPublishedOn.toString());
-        }
-        data.put("lastsubmittedon", lastSubmittedOn);
-        data.put("lastupdatedon", lastUpdatedOn);
-        data.put("lastpublishedon", lastPublishedOn);
-        return data;
-    }
-
-
-    private Map getEpochConvertedDialcodeDataMap(Map data) {
-
-        Object generatedOn = data.get("generatedon");
-        Object publishedOn = data.get("publishedon");
-        if (generatedOn instanceof String) {
-            generatedOn = getConvertedTimestamp(generatedOn.toString());
-        }
-        if (publishedOn instanceof String) {
-            publishedOn = getConvertedTimestamp(publishedOn.toString());
-        }
-        data.put("generatedon", generatedOn);
-        data.put("publishedon", publishedOn);
-        return data;
-    }
-}
-
-
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java
deleted file mode 100644
index a2e64276db..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/domain/UserDataUpdater.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.task.DeNormalizationConfig;
-import org.ekstep.ep.samza.util.UserDataCache;
-
-import java.util.Map;
-
-public class UserDataUpdater extends IEventUpdater {
-
-    private UserDataCache userDataCache;
-
-    UserDataUpdater(UserDataCache userDataCache) {
-        this.userDataCache = userDataCache;
-        this.cacheType = "user";
-    }
-
-    public void update(Event event) {
-        Map<String, Object> userCacheData;
-        String userId = event.actorId();
-        if (userId != null && !userId.isEmpty() && !"system".equalsIgnoreCase(event.actorType())) {
-            userCacheData = userDataCache.getUserData(userId);
-            if (userCacheData != null && !userCacheData.isEmpty()) {
-                event.addUserData(userCacheData);
-            } else {
-                event.setFlag(DeNormalizationConfig.getUserLocationJobFlag(), false);
-            }
-        }
-    }
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java
deleted file mode 100644
index a2cd931e1e..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/service/DeNormalizationService.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.domain.EventUpdaterFactory;
-import org.ekstep.ep.samza.task.DeNormalizationConfig;
-import org.ekstep.ep.samza.task.DeNormalizationSink;
-import org.ekstep.ep.samza.task.DeNormalizationSource;
-
-import java.util.List;
-
-public class DeNormalizationService {
-
-    private static Logger LOGGER = new Logger(DeNormalizationService.class);
-    private final DeNormalizationConfig config;
-    private final EventUpdaterFactory eventUpdaterFactory;
-
-    public DeNormalizationService(DeNormalizationConfig config, EventUpdaterFactory eventUpdaterFactory) {
-        this.config = config;
-        this.eventUpdaterFactory = eventUpdaterFactory;
-    }
-
-    public void process(DeNormalizationSource source, DeNormalizationSink sink) {
-        try {
-            Event event = source.getEvent();
-            String eid = event.eid();
-            List<String> summaryRouteEventPrefix = this.config.getSummaryFilterEvents();
-
-            if("ERROR".equals(eid)){
-                LOGGER.debug(null,"Skipping denormalization as eid is ERROR");
-                sink.toSuccessTopic(event);
-                return;
-            }
-
-            if (summaryRouteEventPrefix.contains(eid)) {
-                denormEvent(event);
-                sink.toSuccessTopic(event);
-            } else if (eid.startsWith("ME_")) {
-                LOGGER.debug(null, "Ignoring as eid is other than WFS");
-                sink.incrementSkippedCount(event);
-            } else {
-                // ignore past data (older than last X months)
-                if (event.isOlder(config.ignorePeriodInMonths())) {
-                    sink.incExpiredEventCount();
-                    LOGGER.debug(null, "Ignoring as ets is older than N months");
-                    sink.toFailedTopic(event, "older than " + config.ignorePeriodInMonths() + " months");
-                } else {
-                    denormEvent(event);
-                    sink.toSuccessTopic(event);
-                }
-            }
-        } catch(JsonSyntaxException e){
-        	e.printStackTrace();
-            LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-            sink.toMalformedTopic(source.getMessage());
-        }
-    }
-
-    private void denormEvent(Event event) {
-
-        // correct future dates
-        event.compareAndAlterEts();
-
-        if ("dialcode".equalsIgnoreCase(event.objectType()) || "qr".equalsIgnoreCase(event.objectType())) {
-            // add dialcode details to the event where object.type = dialcode/qr
-            eventUpdaterFactory.getInstance("dialcode-data-updater").update(event, event.getKey("content"));
-        } else if ("user".equalsIgnoreCase(event.objectType())) {
-            // skipping since it is same as user-denorm
-        }
-        else {
-            // add content details to the event
-            eventUpdaterFactory.getInstance("content-data-updater").update(event, event.getKey("content"));
-            if(event.checkObjectIdNotEqualsRollUpl1Id()) {
-                eventUpdaterFactory.getInstance("collection-data-updater").update(event, event.getKey("collection"));
-            }
-        }
-        // add user details to the event
-        eventUpdaterFactory.getInstance("user-data-updater").update(event);
-
-        // add device details to the event
-//        eventUpdaterFactory.getInstance("device-data-updater").update(event);
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java
deleted file mode 100644
index 2372629928..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationConfig.java
+++ /dev/null
@@ -1,93 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DeNormalizationConfig {
-
-    private final String JOB_NAME = "DeNormalization";
-    private static final String deviceDataJobFlag = "device_data_retrieved";
-    private static final String userDataJobFlag = "user_data_retrieved";
-    private static final String contentDataJobFlag = "content_data_retrieved";
-    private static final String dialCodeDataJobFlag = "dialcode_data_retrieved";
-    private static final String collectionDataJobFlag = "collection_data_retrieved";
-
-    private String successTopic;
-    private String failedTopic;
-    private String malformedTopic;
-    private Integer ignorePeriodInMonths;
-    private List<String> summaryFilterEvents;
-
-
-    public DeNormalizationConfig(Config config) {
-        successTopic = config.get("output.success.topic.name", "telemetry.denorm");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        ignorePeriodInMonths = config.getInt("telemetry.ignore.period.months", 6);
-        List<String> defaultSummaryEvents = new ArrayList<String>();
-        defaultSummaryEvents.add("ME_WORKFLOW_SUMMARY");
-        summaryFilterEvents = config.getList("summary.filter.events", defaultSummaryEvents);
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public Integer ignorePeriodInMonths() {
-        return ignorePeriodInMonths;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public static String getJobFlag(String type) {
-        switch (type) {
-            case "device":
-                return deviceDataJobFlag;
-            case "user":
-                return userDataJobFlag;
-            case "content":
-                return contentDataJobFlag;
-            case "dialcode":
-                return dialCodeDataJobFlag;
-            case "collection":
-                return collectionDataJobFlag;
-            default:
-                return "";
-        }
-    }
-
-    public static String getDeviceLocationJobFlag() {
-        return deviceDataJobFlag;
-    }
-
-    public static String getUserLocationJobFlag() {
-        return userDataJobFlag;
-    }
-
-    public static String getContentLocationJobFlag() {
-        return contentDataJobFlag;
-    }
-
-    public static String getCollectionLocationJobFlag() { return collectionDataJobFlag; }
-    public static String getDialCodeLocationJobFlag() {
-        return dialCodeDataJobFlag;
-    }
-
-    public List<String> getSummaryFilterEvents() {
-        return this.summaryFilterEvents;
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java
deleted file mode 100644
index b385c8ff7f..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSink.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class DeNormalizationSink extends BaseSink {
-
-	private DeNormalizationConfig config;
-
-	public DeNormalizationSink(MessageCollector collector, JobMetrics metrics, DeNormalizationConfig config) {
-		super(collector, metrics);
-		this.config = config;
-	}
-
-	public void toSuccessTopic(Event event) {
-		toTopic(config.successTopic(), event.did(), event.getJson());
-		metrics.incSuccessCounter();
-	}
-
-	public void toFailedTopic(Event event, String failedMessage) {
-		event.markFailure(failedMessage, config);
-		toTopic(config.failedTopic(), event.did(), event.getJson());
-		metrics.incFailedCounter();
-	}
-
-	public void toMalformedTopic(String message) {
-		toTopic(config.malformedTopic(), null, message);
-		metrics.incErrorCounter();
-	}
-
-	public void incrementSkippedCount(Event event) {
-		metrics.incSkippedCounter();
-	}
-
-	public void incExpiredEventCount() {
-		metrics.incExpiredEventCount();
-	}
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java
deleted file mode 100644
index 59a2c03b2a..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationSource.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import java.util.Map;
-
-public class DeNormalizationSource {
-	static Logger LOGGER = new Logger(DeNormalizationSource.class);
-
-	private IncomingMessageEnvelope envelope;
-
-	public DeNormalizationSource(IncomingMessageEnvelope envelope) {
-		this.envelope = envelope;
-	}
-
-	public Event getEvent() {
-		String message = (String) envelope.getMessage();
-		@SuppressWarnings("unchecked")
-		Map<String, Object> jsonMap = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-		return new Event(jsonMap);
-	}
-
-	public String getMessage() {
-		return envelope.toString();
-	}
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java
deleted file mode 100644
index 044e6354df..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/task/DeNormalizationTask.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.EventUpdaterFactory;
-import org.ekstep.ep.samza.service.DeNormalizationService;
-import org.ekstep.ep.samza.util.ContentDataCache;
-import org.ekstep.ep.samza.util.DialCodeDataCache;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.util.UserDataCache;
-
-public class DeNormalizationTask extends BaseSamzaTask {
-
-    private DeNormalizationConfig config;
-    private UserDataCache userCache;
-    private ContentDataCache contentCache;
-    private DialCodeDataCache dialcodeCache;
-    private JobMetrics metrics;
-    private RedisConnect redisConnect;
-    private DeNormalizationService service;
-
-    public DeNormalizationTask(Config config, TaskContext context, UserDataCache userCache,
-                               ContentDataCache contentCache, DialCodeDataCache dialcodeCache, JobMetrics jobMetrics, RedisConnect redisConnect) {
-        init(config, context, userCache, contentCache, dialcodeCache, jobMetrics, redisConnect);
-    }
-
-    public DeNormalizationTask() {
-
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) {
-        init(config, context, userCache, contentCache, dialcodeCache, metrics, redisConnect);
-    }
-
-
-    public void init(Config config, TaskContext context, UserDataCache userCache, ContentDataCache contentCache,
-                     DialCodeDataCache dialcodeCache, JobMetrics jobMetrics, RedisConnect redisConnect) {
-
-        this.config = new DeNormalizationConfig(config);
-        this.metrics = jobMetrics == null ? new JobMetrics(context, this.config.jobName()) : jobMetrics;
-        this.userCache = userCache == null ? new UserDataCache(config, metrics, redisConnect) : userCache;
-        this.contentCache = contentCache == null ? new ContentDataCache(config, metrics) : contentCache;
-        this.dialcodeCache = dialcodeCache == null ? new DialCodeDataCache(config, metrics) : dialcodeCache;
-        service = new DeNormalizationService(this.config, new EventUpdaterFactory(this.contentCache, this.userCache, this.dialcodeCache));
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) {
-        DeNormalizationSource source = new DeNormalizationSource(envelope);
-        DeNormalizationSink sink = new DeNormalizationSink(collector, metrics, config);
-        service.process(source, sink);
-    }
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java
deleted file mode 100644
index 998b1eec6d..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/ContentDataCache.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-public class ContentDataCache extends DataCache {
-
-    public ContentDataCache(Config config, JobMetrics metrics) {
-
-        super(config.getList("content.metadata.fields", Arrays.asList("name", "objectType", "contentType", "mediaType", "language", "medium", "mimeType", "createdBy", "createdFor", "framework", "board", "subject", "status", "pkgVersion", "lastSubmittedOn", "lastUpdatedOn", "lastPublishedOn")));
-        this.databaseIndex = config.getInt("redis.contentDB.index", 5);
-        this.redisConnect = new RedisConnect(config);
-        this.redisConnection = this.redisConnect.getConnection(databaseIndex);
-        this.metrics = metrics;
-    }
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java
deleted file mode 100644
index 8219ff22c5..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DataCache.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-import java.lang.reflect.Type;
-import java.util.*;
-
-public class DataCache {
-
-    private static Logger LOGGER = new Logger(DataCache.class);
-
-    protected RedisConnect redisConnect;
-    protected int databaseIndex;
-    protected Jedis redisConnection;
-    protected JobMetrics metrics;
-    protected List fieldsList;
-    private Gson gson = new Gson();
-
-    public DataCache(List fieldsList) {
-        this.fieldsList = fieldsList;
-    }
-
-    public Map<String, Object> getData(String key) {
-        Map<String, Object> cacheDataMap;
-        try {
-            cacheDataMap = getDataFromCache(key);
-        } catch (JedisException ex) {
-            LOGGER.error("", "Exception when retrieving data from redis cache ", ex);
-            
-            this.redisConnection.close();
-            this.redisConnection = redisConnect.getConnection(databaseIndex);
-            cacheDataMap = getDataFromCache(key);
-        }
-        if (cacheDataMap != null && !cacheDataMap.isEmpty()) {
-        	metrics.incCacheHitCounter();
-        }
-        return cacheDataMap;
-    }
-
-    private Map<String, Object> getDataFromCache(String key) {
-        Map<String, Object> cacheData = new HashMap<>();
-        String dataNode = redisConnection.get(key);
-        if(dataNode != null && !dataNode.isEmpty()) {
-            Type type = new TypeToken<Map<String, Object>>() {}.getType();
-            Map<String, Object> parsedData = gson.fromJson(dataNode, type);
-            parsedData.keySet().retainAll(fieldsList);
-            parsedData.values().removeAll(Collections.singleton(""));
-            for (Map.Entry<String, Object> entry : parsedData.entrySet()) {
-                cacheData.put(entry.getKey().toLowerCase().replace("_", ""), entry.getValue());
-            }
-        }
-        return cacheData;
-    }
-
-    public List<Map> getData(List<String> keys) {
-        List<Map> list = new ArrayList<>();
-        for (String entry : keys) {
-            Map data = getData(entry);
-            if (data != null && !data.isEmpty()) {
-                list.add(data);
-            }
-        }
-        return list;
-    }
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java
deleted file mode 100644
index a9e988b74d..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/DialCodeDataCache.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-import java.util.Arrays;
-
-public class DialCodeDataCache extends DataCache {
-
-
-    public DialCodeDataCache(Config config, JobMetrics metrics) {
-
-        super(config.getList("dialcode.metadata.fields", Arrays.asList("identifier", "channel", "batchcode", "publisher", "generated_on", "published_on", "status")));
-        this.databaseIndex = config.getInt("redis.dialcodeDB.index", 6);
-        this.redisConnect = new RedisConnect(config);
-        this.redisConnection = this.redisConnect.getConnection(databaseIndex);
-        this.metrics = metrics;
-    }
-
-}
diff --git a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java b/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java
deleted file mode 100644
index 86471c8e55..0000000000
--- a/data-pipeline/de-normalization/src/main/java/org/ekstep/ep/samza/util/UserDataCache.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-
-import java.lang.reflect.Type;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-public class UserDataCache extends DataCache {
-
-    private static Logger LOGGER = new Logger(UserDataCache.class);
-
-    private RedisConnect redisPool;
-    private Jedis redisConnection;
-    private String userSignInTypeDefault;
-    private String userLoginInTypeDefault;
-    private Type mapType = new TypeToken<Map<String, Object>>() {
-    }.getType();
-    private Gson gson = new Gson();
-    private JobMetrics metrics;
-    private int databaseIndex;
-
-
-    public UserDataCache(Config config, JobMetrics metrics, RedisConnect redisConnect) {
-
-        super(config.getList("user.metadata.fields", Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype")));
-        this.metrics = metrics;
-        this.databaseIndex = config.getInt("redis.userDB.index", 4);
-        this.redisPool = null == redisConnect ? new RedisConnect(config) : redisConnect;
-        this.redisConnection = this.redisPool.getConnection(databaseIndex);
-        this.userSignInTypeDefault = config.get("user.signin.type.default", "Anonymous");
-        this.userLoginInTypeDefault = config.get("user.login.type.default", "NA");
-    }
-
-    public Map<String, Object> getUserData(String userId) {
-        if ("anonymous".equalsIgnoreCase(userId)) return null;
-        Map<String, Object> userDataMap;
-        try {
-            userDataMap = getUserDataFromCache(userId);
-        } catch (JedisException ex) {
-            this.redisConnection.close();
-            this.redisConnection = redisPool.getConnection(databaseIndex);
-            userDataMap = getUserDataFromCache(userId);
-        }
-        if (null != userDataMap && !userDataMap.isEmpty()) {
-            userDataMap.keySet().retainAll(this.fieldsList);
-            metrics.incUserCacheHitCount();
-        }
-        userDataMap = getUserSigninLoginDetails(userDataMap);
-
-        if (userDataMap.size() <= 2) {  //Since SigninType and LoginType are default values, incrementing no data metric only if other user details are not present
-            metrics.incEmptyCacheValueCounter();
-        }
-        return userDataMap;
-    }
-
-    private Map<String, Object> getUserSigninLoginDetails(Map<String, Object> userDataMap) {
-        Map<String,Object> userMap = null!=userDataMap ? userDataMap : new HashMap<>();
-        if (!userMap.containsKey("usersignintype")) {
-            userMap.put("usersignintype", userSignInTypeDefault);
-        }
-        if (!userMap.containsKey("userlogintype"))
-            userMap.put("userlogintype", userLoginInTypeDefault);
-        return userMap;
-    }
-
-    private Map<String, Object> getUserDataFromCache(String userId) {
-        Map<String, Object> cacheData = new HashMap<>();
-        String data = redisConnection.get(userId);
-        if (data != null && !data.isEmpty()) {
-            cacheData = gson.fromJson(data, mapType);
-        }
-        return cacheData;
-    }
-}
diff --git a/data-pipeline/de-normalization/src/main/resources/log4j.xml b/data-pipeline/de-normalization/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/de-normalization/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java
deleted file mode 100644
index bddf1087f8..0000000000
--- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/ContentDataUpdaterTest.java
+++ /dev/null
@@ -1,114 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.gson.Gson;
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.util.ContentDataCache;
-import org.ekstep.ep.samza.util.DataCache;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import redis.clients.jedis.Jedis;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.stub;
-
-public class ContentDataUpdaterTest {
-
-    private Config configMock;
-    private RedisConnect redisConnectMock;
-    private ContentDataUpdater contentDataUpdater;
-    private ContentDataCache dataCacheMock;
-    private Jedis jedisMock;
-
-    @SuppressWarnings("unchecked")
-    @Before
-    public void setUp() {
-
-        jedisMock = new MockJedis("test");
-        configMock = mock(Config.class);
-        redisConnectMock = mock(RedisConnect.class);
-        jedisMock.flushAll();
-    }
-
-    @Test
-    public void shouldUpdateEventWithConvertedTimeFields() throws Exception {
-        Map<String, Object> data = new HashMap();
-        data.put("lastupdatedon", "2019-02-12T11:35:35.919+0000");
-        data.put("lastsubmittedon", "2019-02-12T11:35:35.919+0000");
-        data.put("lastpublishedon", "2019-02-12T11:35:35.919");
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        dataCacheMock = mock(ContentDataCache.class);
-        contentDataUpdater = new ContentDataUpdater(dataCacheMock);
-        stub(dataCacheMock.getData("test-content-1")).toReturn(data);
-
-        Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0");
-        objectMap.put("type", "Content");
-        Map eventMap = new HashMap();
-        eventMap.put("object", objectMap);
-        Event input = new Event(eventMap);
-        contentDataUpdater.update(input, "test-content-1");
-        String output = input.toString();
-        assertTrue(output.contains("lastsubmittedon=1549971335919"));
-        assertTrue(output.contains("lastpublishedon=1549971335919"));
-        assertTrue(output.contains("lastupdatedon=1549971335919"));
-    }
-
-    @Test
-    public void shouldUpdateEventWithOutConvertion() throws Exception {
-        Map<String, Object> data = new HashMap();
-        data.put("lastupdatedon", 1549971335919L);
-        data.put("lastsubmittedon", 1549971335919L);
-        data.put("lastpublishedon", 1.549971335919E12);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        dataCacheMock = mock(ContentDataCache.class);
-        contentDataUpdater = new ContentDataUpdater(dataCacheMock);
-        stub(dataCacheMock.getData("test-content-1")).toReturn(data);
-
-        Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0");
-        objectMap.put("type", "Content");
-        Map eventMap = new HashMap();
-        eventMap.put("object", objectMap);
-        Event input = new Event(eventMap);
-        contentDataUpdater.update(input, "test-content-1");
-        String output = input.toString();
-        assertTrue(output.contains("lastsubmittedon=1549971335919"));
-        assertTrue(output.contains("lastpublishedon=1.549971335919E12"));
-        assertTrue(output.contains("lastupdatedon=1549971335919"));
-    }
-
-    @Test
-    public void shouldUpdateEventWith0ForIncorrectDateFormat() throws Exception {
-        Map<String, Object> data = new HashMap();
-        data.put("lastupdatedon", "2019-02-12");
-        data.put("lastsubmittedon", "2019-02-12");
-        data.put("lastpublishedon", "2019-02-12");
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        dataCacheMock = mock(ContentDataCache.class);
-        contentDataUpdater = new ContentDataUpdater(dataCacheMock);
-        stub(dataCacheMock.getData("test-content-1")).toReturn(data);
-
-        Map objectMap = new HashMap(); objectMap.put("id", "test-content-1"); objectMap.put("ver", "1.0");
-        objectMap.put("type", "Content");
-        Map eventMap = new HashMap();
-        eventMap.put("object", objectMap);
-        Event input = new Event(eventMap);
-        contentDataUpdater.update(input, "test-content-1");
-        String output = input.toString();
-        assertTrue(output.contains("lastsubmittedon=0"));
-        assertTrue(output.contains("lastpublishedon=0"));
-        assertTrue(output.contains("lastupdatedon=0"));
-    }
-
-
-}
diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
deleted file mode 100644
index 1107365656..0000000000
--- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
+++ /dev/null
@@ -1,112 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import org.joda.time.DateTime;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-public class EventTest {
-
-    @Test
-    public void shouldUpdateEventWithAlteredETSField() throws Exception {
-
-        Map eventMap = new HashMap();
-        Long ets = new DateTime().plusDays(2).getMillis();
-        eventMap.put("ets", ets);
-        Event input = new Event(eventMap);
-        Long output = input.compareAndAlterEts();
-        assertTrue(output < ets);
-    }
-
-    @Test
-    public void shouldNotUpdateEventWithETSField() throws Exception {
-
-        Map eventMap = new HashMap();
-        Long ets = new DateTime().getMillis();
-        eventMap.put("ets", ets);
-        Event input = new Event(eventMap);
-        Long output = input.compareAndAlterEts();
-        assertTrue(output == ets);
-    }
-
-    @Test
-    public void shouldReturnTrueForOlderData() throws Exception {
-
-        Map eventMap = new HashMap();
-        Long ets = new DateTime().minusMonths(7).getMillis();
-        eventMap.put("ets", ets);
-        Event input = new Event(eventMap);
-        assertTrue(input.isOlder(6));
-    }
-
-    @Test
-    public void shouldReturnFalseForOlderData() throws Exception {
-
-        Map eventMap = new HashMap();
-        Long ets = new DateTime().minusMonths(3).getMillis();
-        eventMap.put("ets", ets);
-        Event input = new Event(eventMap);
-        assertFalse(input.isOlder(6));
-    }
-
-    @Test
-    public void shouldReturnLatestVersion() throws Exception {
-
-        Map eventMap = new HashMap();
-        eventMap.put("ver", "3.0");
-        Event input = new Event(eventMap);
-        String updatedVer = input.getUpgradedVersion();
-        assertTrue(updatedVer.equals("3.1"));
-    }
-
-    @Test
-    public void shouldReturnSummaryLatestVersion() throws Exception {
-
-        Map eventMap = new HashMap();
-        eventMap.put("ver", "2.1");
-        Event input = new Event(eventMap);
-        String updatedVer = input.getUpgradedVersion();
-        assertTrue(updatedVer.equals("2.2"));
-    }
-
-    @Test
-    public void ShouldaddISOStateCodeToDeviceData() {
-
-        String DEVICE_PROFILE_DETAILS = "{\n" +
-                "\"fcm_token\" : \"\",\n" +
-                "\"city\" : \"Bengaluru\",\n" +
-                "\"device_id\" : \"232455\",\n" +
-                "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" +
-                "\"state\" : \"Karnataka\",\n" +
-                "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" +
-                "\"country\" : \"India\",\n" +
-                "\"country_code\" : \"IN\",\n" +
-                "\"producer_id\" : \"dev.sunbird.portal\",\n" +
-                "\"state_code_custom\" : 29,\n" +
-                "\"state_code\" : \"KA\",\n" +
-                "\"state_custom\" : \"Karnataka\",\n" +
-                "\"district_custom\" : \"Karnataka\",\n" +
-                "\"first_access\": 1568377184000,\n" +
-                "\"api_last_updated_on\": 1568377184000,\n" +
-                "\"user_declared_district\" : \"Bengaluru\",\n" +
-                "\"user_declared_state\" : \"Karnataka\"\n" +
-                "}";
-
-        Event event = new Event(new Gson().fromJson(DEVICE_PROFILE_DETAILS, Map.class));
-        Map deviceData = new HashMap();
-        deviceData.put("first_access", "1568377184000");
-        deviceData.put("device_id", "232455");
-        deviceData.put("statecode", "KA");
-        event.addDeviceData(deviceData);
-        Map result = event.addISOStateCodeToDeviceData(deviceData);
-        Assert.assertEquals(result.get("iso3166statecode"), "IN-KA");
-
-    }
-
-}
diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index 55bfed6e2a..0000000000
--- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,1421 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import org.joda.time.DateTime;
-
-import java.util.Map;
-
-public class EventFixture {
-
-    public static Long current_ets = new DateTime().getMillis();
-
-    public static final String UNPARSABLE_START_EVENT = "{\n" +
-            "  \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"mode\": \"WIFI\",\n" +
-            "      \"ver\": \"12\",\n" +
-            "      \"size\": 12.67,\n" +
-            "      \"err\": \"\",\n" +
-            "      \"referrer\": [\n" +
-            "        {\n" +
-            "          \"action\": \"INSTALL\",\n" +
-            "          \"utmsource\": \"Ekstep\",\n" +
-            "          \"utmmedium\": \"Portal\",\n" +
-            "          \"utmterm\": \"December 2016\",\n" +
-            "          \"utmcontent\": \"Ramayana\",\n" +
-            "          \"utmcampaign\": \"Epics of India\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"START\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"1.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.0\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    ";
-
-    public static final String INTERACT_EVENT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"do_31249561779090227216256\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"version\": \"\"\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITH_OBJECT_ROLLUP = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"object\": {\n" +
-                "        \"id\": \"do_31277438304183091217888\",\n" +
-            "        \"type\": \"TextBook\",\n" +
-            "        \"version\": \"\",\n" +
-            "        \"rollup\": { \n" +
-            "           \"l1\": \"do_31249561779090227216256\"\n" +
-            "       }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITH_OBJECTID_EQUALS_ROLLUPID = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"do_31277438304183091217888\",\n" +
-            "        \"type\": \"TextBook\",\n" +
-            "        \"version\": \"\",\n" +
-            "        \"rollup\": { \n" +
-            "           \"l1\": \"do_31277438304183091217888\"\n" +
-            "       }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITH_DEVICEDATA = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "        \"id\": \"draftContentId\",\n" +
-            "        \"type\": \"click\",\n" +
-            "        \"pageid\": \"DraftContent\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.11.0\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"staging.diksha.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"workspace\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"sid\": \"7ALbNUfErYtkGqPdyK0kjfRSETJXdKp4\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"012550822176260096119\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"INTERACT:4b645cf3d2a587413603de219eece5ed\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"do_21269871962088243212780\",\n" +
-            "        \"type\": \"draft\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"012550822176260096119\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1.550140278251E12,\n" +
-            "    \"@timestamp\": \"2019-02-14T10:31:18.251Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"device_location_retrieved\": true,\n" +
-            "        \"user_location_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-14T10:23:54.456+0000\",\n" +
-            "    \"devicedata\": {\n" +
-            "        \"country\": \"India\",\n" +
-            "        \"city\": \"Bengaluru\",\n" +
-            "        \"countrycode\": \"IN\",\n" +
-            "        \"state\": \"Karnataka\",\n" +
-            "        \"statecode\": \"KA\"\n" +
-            "    },\n" +
-            "    \"userdata\": {\n" +
-            "        \"district\": \"\",\n" +
-            "        \"state\": \"\"\n" +
-            "    }\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITH_EMPTY_LOC = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "        \"id\": \"draftContentId\",\n" +
-            "        \"type\": \"click\",\n" +
-            "        \"pageid\": \"DraftContent\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"60c02e78-64f9-400c-be3f-d91256d58cf1\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.11.0\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"staging.diksha.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"workspace\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"sid\": \"7ALbNUfErYtkGqPdyK0kjfRSETJXdKp4\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"012550822176260096119\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"INTERACT:4b645cf3d2a587413603de219eece5ed\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"do_21269871962088243212780\",\n" +
-            "        \"type\": \"draft\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"012550822176260096119\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1.550140278251E12,\n" +
-            "    \"@timestamp\": \"2019-02-14T10:31:18.251Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"device_location_retrieved\": true,\n" +
-            "        \"user_location_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-14T10:23:54.456+0000\",\n" +
-            "    \"devicedata\": {\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"countrycode\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"statecode\": \"\"\n" +
-            "    },\n" +
-            "    \"userdata\": {\n" +
-            "        \"district\": \"\",\n" +
-            "        \"state\": \"\"\n" +
-            "    }\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITHOUT_DID = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"do_31249561779090227216256\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"version\": \"\"\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"System\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"do_31249561779090227216256\",\n" +
-            "        \"type\": \"Content\",\n" +
-            "        \"version\": \"\"\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String INTERACT_EVENT_WITHOUT_OBJECT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "    },\n" +
-            "    \"eid\": \"INTERACT\",\n" +
-            "    \"edata\": {\n" +
-            "		 \"loc\":\"xyz\",\n" +
-            "        \"id\": \"ContentDetail\",\n" +
-            "        \"pageid\": \"ContentDetail\",\n" +
-            "        \"type\": \"TOUCH\",\n" +
-            "        \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"2.1.8\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"env\": \"sdk\",\n" +
-            "        \"cdata\": [\n" +
-            "            {\n" +
-            "                \"type\": \"qr\",\n" +
-            "                \"id\": \"K4KCXE\"\n" +
-            "            },\n" +
-            "            {\n" +
-            "                \"type\": \"API\",\n" +
-            "                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "            }\n" +
-            "        ],\n" +
-            "        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "    },\n" +
-            "    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"syncts\": 1539846605341,\n" +
-            "    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-            "}", current_ets);
-
-    public static final String SEARCH_EVENT_WITHOUT_DIALCODE = String.format("{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"mid\": \"LP.1543688467595.c66eb854-82c4-4b73-8902-82209be643ed\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 51,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"contentType\": \"Course\",\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ],\n" +
-            "        \"status\": [\n" +
-            "          \"Live\"\n" +
-            "        ],\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {}\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312461520193110016213245\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688467885,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:07.885Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:07.595+0000\"\n" +
-            "  }", current_ets);
-
-    public static final String SEARCH_EVENT_WITH_DIALCODE_AS_STRING = String.format("{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 1,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"dialcodes\": \"8ZEDTP\",\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"contentType\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {},\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312531599251210240213439\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688463882,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" +
-            "  }", current_ets);
-
-    public static final String SEARCH_EVENT_WITH_DIALCODE_AS_LIST = String.format("{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 1,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"dialcodes\": [\"8ZEDTP\", \"4ZEDTP\"],\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"contentType\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {},\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312531599251210240213439\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688463882,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" +
-            "  }", current_ets);
-
-    public static final String IMPRESSION_EVENT_WITH_DIALCODE_AS_OBJECT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"anonymous\"\n" +
-            "    },\n" +
-            "    \"eid\": \"IMPRESSION\",\n" +
-            "    \"edata\": {\n" +
-            "        \"visits\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"type\": \"view\",\n" +
-            "        \"pageid\": \"dialcode\",\n" +
-            "        \"subtype\": \"pageexit\",\n" +
-            "        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"anonymous\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.7.1\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"prod.diksha.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"env\": \"public\",\n" +
-            "        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-            "        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"977D3I\",\n" +
-            "        \"type\": \"dialcode\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1550501698819,\n" +
-            "    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-            "    \"ldata\": {\n" +
-            "        \"country_code\": \"\",\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"state_code\": \"\"\n" +
-            "    }  \n" +
-            "}", current_ets);
-
-    public static final String IMPRESSION_EVENT_WITH_QR_AS_OBJECT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"anonymous\"\n" +
-            "    },\n" +
-            "    \"eid\": \"IMPRESSION\",\n" +
-            "    \"edata\": {\n" +
-            "        \"visits\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"type\": \"view\",\n" +
-            "        \"pageid\": \"dialcode\",\n" +
-            "        \"subtype\": \"pageexit\",\n" +
-            "        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"anonymous\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.7.1\",\n" +
-            "            \"pid\": \"sunbird.app\",\n" +
-            "            \"id\": \"prod.diksha.app\"\n" +
-            "        },\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"env\": \"public\",\n" +
-            "        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-            "        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"977D3I\",\n" +
-            "        \"type\": \"qr\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1550501698819,\n" +
-            "    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-            "    \"ldata\": {\n" +
-            "        \"country_code\": \"\",\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"state_code\": \"\"\n" +
-            "    }  \n" +
-            "}", current_ets);
-
-    public static final String IMPRESSION_EVENT_WITH_DIALCODE_CAMELCASE_AS_OBJECT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"anonymous\"\n" +
-            "    },\n" +
-            "    \"eid\": \"IMPRESSION\",\n" +
-            "    \"edata\": {\n" +
-            "        \"visits\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"type\": \"view\",\n" +
-            "        \"pageid\": \"dialcode\",\n" +
-            "        \"subtype\": \"pageexit\",\n" +
-            "        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"anonymous\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.7.1\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"prod.diksha.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"env\": \"public\",\n" +
-            "        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-            "        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"977D3I\",\n" +
-            "        \"type\": \"DialCode\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1550501698819,\n" +
-            "    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-            "    \"ldata\": {\n" +
-            "        \"country_code\": \"\",\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"state_code\": \"\"\n" +
-            "    }  \n" +
-            "}", current_ets);
-
-    public static final String SEARCH_EVENT_WITH_CAMELCASE_DIALCODE_AS_STRING = String.format("{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 1,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"dialCodes\": \"8ZEDTP\",\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"contentType\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {},\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312531599251210240213439\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688463882,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" +
-            "  }", current_ets);
-
-    public static final String SEARCH_EVENT_WITH_FUTURE_ETS = "{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": 2530937155000,\n" +
-            "    \"ver\": \"2.1\",\n" +
-            "    \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 1,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"dialCodes\": \"8ZEDTP\",\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"contentType\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {},\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312531599251210240213439\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688463882,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" +
-            "  }";
-
-    public static final String SEARCH_EVENT_WITH_OLDER_ETS = "{\n" +
-            "    \"eid\": \"SEARCH\",\n" +
-            "    \"ets\": 1520753300000,\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"mid\": \"LP.1543688463694.670c6cf8-2cd2-45a7-b531-f212ac2847ec\",\n" +
-            "    \"actor\": {\n" +
-            "      \"id\": \"org.ekstep.learning.platform\",\n" +
-            "      \"type\": \"System\"\n" +
-            "    },\n" +
-            "    \"context\": {\n" +
-            "      \"channel\": \"in.ekstep\",\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"prod.ntp.learning.platform\",\n" +
-            "        \"pid\": \"search-service\",\n" +
-            "        \"ver\": \"1.0\"\n" +
-            "      },\n" +
-            "      \"env\": \"search\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"size\": 1,\n" +
-            "      \"query\": \"\",\n" +
-            "      \"filters\": {\n" +
-            "        \"dialCodes\": \"8ZEDTP\",\n" +
-            "        \"channel\": {\n" +
-            "          \"ne\": [\n" +
-            "            \"0124433024890224640\",\n" +
-            "            \"0124446042259128320\",\n" +
-            "            \"0124487522476933120\",\n" +
-            "            \"0125840271570288640\",\n" +
-            "            \"0124453662635048969\"\n" +
-            "          ]\n" +
-            "        },\n" +
-            "        \"framework\": {},\n" +
-            "        \"contentType\": {},\n" +
-            "        \"mimeType\": {},\n" +
-            "        \"resourceType\": {},\n" +
-            "        \"objectType\": [\n" +
-            "          \"Content\",\n" +
-            "          \"ContentImage\"\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      \"sort\": {},\n" +
-            "      \"type\": \"content\",\n" +
-            "      \"topn\": [\n" +
-            "        {\n" +
-            "          \"identifier\": \"do_312531599251210240213439\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"tv_processed\": true,\n" +
-            "      \"dd_processed\": true\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"syncts\": 1543688463882,\n" +
-            "    \"@timestamp\": \"2018-12-01T18:21:03.882Z\",\n" +
-            "    \"ts\": \"2018-12-01T18:21:03.694+0000\"\n" +
-            "  }";
-
-    public static final String WFS_EVENT = String.format("{\n" +
-            "    \"eid\": \"ME_WORKFLOW_SUMMARY\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"syncts\": 1.553952011044E12,\n" +
-            "    \"ver\": \"1.1\",\n" +
-            "    \"mid\": \"8D7C477F841B6F82937BBC6F61E7C2FD\",\n" +
-            "    \"uid\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" +
-            "    \"context\": {\n" +
-            "        \"pdata\": {\n" +
-            "            \"id\": \"AnalyticsDataPipeline\",\n" +
-            "            \"ver\": \"1.0\",\n" +
-            "            \"model\": \"WorkflowSummarizer\"\n" +
-            "        },\n" +
-            "        \"granularity\": \"SESSION\",\n" +
-            "        \"date_range\": {\n" +
-            "            \"from\": 1.553951969817E12,\n" +
-            "            \"to\": 1.553951999398E12\n" +
-            "        },\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ]\n" +
-            "    },\n" +
-            "    \"dimensions\": {\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"id\": \"prod.diksha.app\",\n" +
-            "            \"ver\": \"2.1.45\",\n" +
-            "            \"pid\": \"sunbird.app\"\n" +
-            "        },\n" +
-            "        \"sid\": \"4a58927b-ad4d-491e-a9e2-73b592a60a22\",\n" +
-            "        \"channel\": \"0123221617357783046602\",\n" +
-            "        \"type\": \"session\",\n" +
-            "        \"mode\": \"\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"interact_events_per_min\": 5.0,\n" +
-            "            \"start_time\": 1.553951969817E12,\n" +
-            "            \"interact_events_count\": 5.0,\n" +
-            "            \"item_responses\": [\n" +
-            "                \n" +
-            "            ],\n" +
-            "            \"end_time\": 1.553951999398E12,\n" +
-            "            \"events_summary\": [\n" +
-            "                {\n" +
-            "                    \"id\": \"START\",\n" +
-            "                    \"count\": 4.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"IMPRESSION\",\n" +
-            "                    \"count\": 4.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"INTERACT\",\n" +
-            "                    \"count\": 12.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"SHARE\",\n" +
-            "                    \"count\": 1.0\n" +
-            "                }\n" +
-            "            ],\n" +
-            "            \"page_summary\": [\n" +
-            "                {\n" +
-            "                    \"id\": \"dial-code-scan-result\",\n" +
-            "                    \"type\": \"view\",\n" +
-            "                    \"env\": \"home\",\n" +
-            "                    \"time_spent\": 4.46,\n" +
-            "                    \"visit_count\": 1.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"user-type-selection\",\n" +
-            "                    \"type\": \"search\",\n" +
-            "                    \"env\": \"home\",\n" +
-            "                    \"time_spent\": 6.08,\n" +
-            "                    \"visit_count\": 1.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"content-detail\",\n" +
-            "                    \"type\": \"detail\",\n" +
-            "                    \"env\": \"home\",\n" +
-            "                    \"time_spent\": 11.73,\n" +
-            "                    \"visit_count\": 1.0\n" +
-            "                },\n" +
-            "                {\n" +
-            "                    \"id\": \"qr-code-scanner\",\n" +
-            "                    \"type\": \"view\",\n" +
-            "                    \"env\": \"home\",\n" +
-            "                    \"time_spent\": 7.6,\n" +
-            "                    \"visit_count\": 1.0\n" +
-            "                }\n" +
-            "            ],\n" +
-            "            \"time_diff\": 29.58,\n" +
-            "            \"telemetry_version\": \"3.0\",\n" +
-            "            \"env_summary\": [\n" +
-            "                {\n" +
-            "                    \"env\": \"home\",\n" +
-            "                    \"time_spent\": 29.87,\n" +
-            "                    \"count\": 1.0\n" +
-            "                }\n" +
-            "            ],\n" +
-            "            \"time_spent\": 29.6\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"\",\n" +
-            "        \"type\": \"\"\n" +
-            "    },\n" +
-            "    \"ts\": \"2019-03-31T03:07:04.736+0000\",\n" +
-            "    \"devicedata\": {\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"countrycode\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"statecode\": \"\"\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "        \"dv_processed\": false\n" +
-            "    }\n" +
-            "}", current_ets);
-
-    public static final String DEVICE_SUMMARY_EVENT = String.format("{\n" +
-            "    \"eid\": \"ME_DEVICE_SUMMARY\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"syncts\": 1554101904592,\n" +
-            "    \"ver\": \"1.0\",\n" +
-            "    \"mid\": \"3ACBFD49BAFC74832A44347FF4F1E611\",\n" +
-            "    \"context\": {\n" +
-            "      \"pdata\": {\n" +
-            "        \"id\": \"AnalyticsDataPipeline\",\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"model\": \"DeviceSummary\"\n" +
-            "      },\n" +
-            "      \"granularity\": \"DAY\",\n" +
-            "      \"date_range\": {\n" +
-            "        \"from\": 1554101897620,\n" +
-            "        \"to\": 1554101904209\n" +
-            "      }\n" +
-            "    },\n" +
-            "    \"dimensions\": {\n" +
-            "      \"did\": \"3f2b155788bef740fe741c92b8d80cac\",\n" +
-            "      \"channel\": \"01231711180382208027\"\n" +
-            "    },\n" +
-            "    \"edata\": {\n" +
-            "      \"eks\": {\n" +
-            "        \"firstAccess\": 1553574785315,\n" +
-            "        \"dial_stats\": {\n" +
-            "          \"total_count\": 0,\n" +
-            "          \"success_count\": 0,\n" +
-            "          \"failure_count\": 0\n" +
-            "        },\n" +
-            "        \"content_downloads\": 0,\n" +
-            "        \"contents_played\": 0,\n" +
-            "        \"total_ts\": 6,\n" +
-            "        \"total_launches\": 1,\n" +
-            "        \"unique_contents_played\": 0\n" +
-            "      }\n" +
-            "    },\n" +
-            "    \"ts\": \"2019-04-02T03:07:09.447+0000\",\n" +
-            "    \"devicedata\": {\n" +
-            "      \"statecustomcode\": \"\",\n" +
-            "      \"country\": \"\",\n" +
-            "      \"city\": \"\",\n" +
-            "      \"countrycode\": \"\",\n" +
-            "      \"state\": \"\",\n" +
-            "      \"statecode\": \"\",\n" +
-            "      \"districtcustom\": \"\",\n" +
-            "      \"statecustomname\": \"\"\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "      \"device_location_retrieved\": true\n" +
-            "    }\n" +
-            "  }", current_ets);
-
-    public static final String ANY_STRING = "Hey Samza, Whats Up?";
-    public static final String EMPTY_JSON = "{}";
-
-    public static final String LOG_EVENT = String.format("{\n" +
-            "   \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n" +
-            "\"actor\":{\n" +
-            "\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n" +
-            "\"type\":\"User\"\n" +
-            "},\n" +
-            "\"context\":{\n" +
-            "\"cdata\":[\n" +
-
-            "],\n" +
-            "\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            /*"\"did\":\"923c675274cfbf19fd0402fe4d2c37afd597f0ab\",\n"+*/
-            "\"env\":\"home\",\n" +
-            "\"pdata\":{\n" +
-            "\"id\":\"prod.diksha.app\",\n" +
-            "\"pid\":\"sunbird.app\",\n" +
-            "\"ver\":\"2.1.45\"\n" +
-            "},\n" +
-            "\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n" +
-            "},\n" +
-            "\"edata\":{\n" +
-            "\"level\":\"INFO\",\n" +
-            "\"message\":\"content-detail\",\n" +
-            "\"params\":[\n" +
-            "{\n" +
-            "\"PopupType\":\"automatic\"\n" +
-            "}\n" +
-            "],\n" +
-            "\"type\":\"api_access\"\n" +
-            "},\n" +
-            "\"eid\":\"LOG\",\n" +
-            "\"ets\":%s2,\n" +
-            "\"flags\":{\n" +
-            "\"dd_processed\":true,\n" +
-            "\"tv_processed\":true\n" +
-            "},\n" +
-            "\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n" +
-            "\"syncts\":1.553040001176E12,\n" +
-            "\"tags\":[\n" +
-            "],\n" +
-            "\"type\":\"events\",\n" +
-            "\"ver\":\"3.0\"\n" +
-            "      }", current_ets);
-
-    public static final String ERROR_EVENT = String.format("{" +
-            "   \"eid\":\"ERROR\"," +
-            "   \"ets\":%s," +
-            "   \"ver\":\"3.0\"," +
-            "   \"mid\":\"LP.1553040097857.bf0e4e15-014e-4a22-ba00-e02ff3c38784\"," +
-            "   \"actor\":{" +
-            "      \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\"," +
-            "      \"type\":\"System\"" +
-            "   }," +
-            "   \"context\":{" +
-            "      \"channel\":\"01235953109336064029450\"," +
-            "      \"pdata\":{" +
-            "         \"id\":\"prod.ntp.learning.platform\"," +
-            "         \"pid\":\"learning-service\"," +
-            "         \"ver\":\"1.0\"" +
-            "      }," +
-            "      \"env\":\"framework\"" +
-            "   }," +
-            "   \"edata\":{" +
-            "      \"err\":\"ERR_DATA_NOT_FOUND\"," +
-            "      \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\"," +
-            "      \"errtype\":\"system\"" +
-            "   }," +
-            "   \"flags\":{" +
-            "      \"tv_processed\":true," +
-            "      \"dd_processed\":true" +
-            "   }," +
-            "   \"type\":\"events\"," +
-            "   \"syncts\":1.553040098435E12," +
-            "   \"@timestamp\":\"2019-03-20T00:01:38.435Z\"" +
-            "}", current_ets);
-
-    public static final String TEST_LOG_EVENT = String.format("{\n" +
-            "   \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n" +
-            "\"actor\":{\n" +
-            "\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n" +
-            "\"type\":\"User\"\n" +
-            "},\n" +
-            "\"context\":{\n" +
-            "\"cdata\":[\n" +
-
-            "],\n" +
-            "\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "\"env\":\"home\",\n" +
-            "\"pdata\":{\n" +
-            "\"id\":\"prod.diksha.app\",\n" +
-            "\"pid\":\"sunbird.app\",\n" +
-            "\"ver\":\"2.1.45\"\n" +
-            "},\n" +
-            "\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n" +
-            "},\n" +
-            "\"edata\":{\n" +
-            "\"level\":\"INFO\",\n" +
-            "\"message\":\"content-detail\",\n" +
-            "\"params\":[\n" +
-            "{\n" +
-            "\"PopupType\":\"automatic\"\n" +
-            "}\n" +
-            "],\n" +
-            "\"type\":\"view\"\n" +
-            "},\n" +
-            "\"eid\":\"LOG\",\n" +
-            "\"ets\":%s,\n" +
-            "\"flags\":{\n" +
-            "\"dd_processed\":true,\n" +
-            "\"tv_processed\":true\n" +
-            "},\n" +
-            "\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n" +
-            "\"syncts\":1.553040001176E12,\n" +
-            "\"tags\":[\n" +
-            "],\n" +
-            "\"type\":\"events\",\n" +
-            "\"ver\":\"3.0\"\n" +
-            "      }", current_ets);
-
-    public static final String AUDIT_EVENT = String.format("{\n" +
-            "    \"actor\": {\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" +
-            "        \"type\": \"User\"\n" +
-            "    },\n" +
-            "    \"eid\": \"AUDIT\",\n" +
-            "    \"edata\": {\n" +
-            "        \"state\": \"Create\",\n" +
-            "        \"prevstate\": \"\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"metadata\": {\n" +
-            "        \"checksum\": \"AUDIT:18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" +
-            "        \"source_eid\": \"GE_CREATE_USER\",\n" +
-            "        \"source_mid\": \"18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" +
-            "        \"index_name\": \"telemetry-2019.06\",\n" +
-            "        \"index_type\": \"events_v1\"\n" +
-            "    },\n" +
-            "    \"@timestamp\": \"2019-06-02T17:21:35.546Z\",\n" +
-            "    \"ets\": %s,\n" +
-            "    \"context\": {\n" +
-            "        \"channel\": \"in.tnpilot\",\n" +
-            "        \"env\": \"Genie\",\n" +
-            "        \"sid\": \"54ff57dd-0f65-4001-90df-a2eed195bb86\",\n" +
-            "        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"id\": \"tnpilot\",\n" +
-            "            \"pid\": \"tnpilot\",\n" +
-            "            \"ver\": \"1.0.25\"\n" +
-            "        },\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ]\n" +
-            "    },\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"v2_converted\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_obtained\": false,\n" +
-            "        \"ldata_processed\": true,\n" +
-            "        \"odn_skipped\": true\n" +
-            "    },\n" +
-            "    \"mid\": \"AUDIT:18413285-6bb0-40ed-b9d1-44a184a3a625\",\n" +
-            "    \"object\": {\n" +
-            "        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\",\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"parent\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \n" +
-            "    ],\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-06-02T17:07:37.141+0000\"\n" +
-            "}", current_ets);
-
-    public static String IMPRESSION_EVENT = String.format("{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"e9d73b7b-211a-43f4-8c01-cc9333757a9d\"\n" +
-            "  },\n" +
-            "  \"eid\": \"IMPRESSION\",\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"view\",\n" +
-            "    \"pageid\": \"splash\",\n" +
-            "    \"uri\": \"splash\"\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": %s,\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"sunbird.app\",\n" +
-            "      \"ver\": \"2.1.91\",\n" +
-            "      \"id\": \"prod.diksha.app\"\n" +
-            "    },\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"env\": \"home\",\n" +
-            "    \"did\": \"f3b08a7848b85f50b611570e547093bae1e30f95\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"sid\": \"3e2f8b9c-1b19-4d5e-8c52-7e7ca1b24fdc\"\n" +
-            "  },\n" +
-            "  \"mid\": \"d07a626f-b1aa-48fd-89b4-73eb53012a0a\",\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"\",\n" +
-            "    \"type\": \"\",\n" +
-            "    \"version\": \"\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"syncts\": 1562146974715,\n" +
-            "  \"@timestamp\": \"2019-07-03T09:42:54.715Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}", current_ets);
-
-    public static Map<String, Object> getMap(String message) {
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-}
diff --git a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java b/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java
deleted file mode 100644
index cb9b186d3a..0000000000
--- a/data-pipeline/de-normalization/src/test/java/org/ekstep/ep/samza/task/DeNormalizationTaskTest.java
+++ /dev/null
@@ -1,728 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.*;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.invocation.InvocationOnMock;
-import redis.clients.jedis.Jedis;
-
-import java.lang.reflect.Type;
-import java.util.*;
-
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class DeNormalizationTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.with_denorm";
-    private static final String FAILED_TOPIC = "telemetry.failed";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private static final Integer ignorePeriodInMonths = 6;
-
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private DeNormalizationTask deNormalizationTask;
-
-    private ContentDataCache contentCacheMock;
-    private DialCodeDataCache dailcodeCacheMock;
-    private JobMetrics jobMetrics;
-    private Jedis jedisMock = new MockJedis("test");
-    private Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-
-    @SuppressWarnings("unchecked")
-    @Before
-    public void setUp() {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = mock(TaskContext.class);
-        metricsRegistry = mock(MetricsRegistry.class);
-        counter = mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = mock(Config.class);
-        RedisConnect redisConnectMock = mock(RedisConnect.class);
-        contentCacheMock = mock(ContentDataCache.class);
-        dailcodeCacheMock = mock(DialCodeDataCache.class);
-        jobMetrics = mock(JobMetrics.class);
-
-
-        stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC);
-        stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-        stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-        stub(configMock.getInt("telemetry.ignore.period.months", ignorePeriodInMonths)).toReturn(ignorePeriodInMonths);
-        stub(configMock.getList("user.metadata.fields",
-                Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype")))
-                .toReturn(Arrays.asList("usertype", "grade", "language", "subject", "state", "district", "usersignintype", "userlogintype"));
-        stub(configMock.getInt("redis.userDB.index", 4)).toReturn(4);
-        stub(redisConnectMock.getConnection(4)).toReturn(jedisMock);
-        List<String> defaultSummaryEvents = new ArrayList<>();
-        defaultSummaryEvents.add("ME_WORKFLOW_SUMMARY");
-        stub(configMock.getList("summary.filter.events", defaultSummaryEvents)).toReturn(defaultSummaryEvents);
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.with_location", new Partition(1)));
-        stub(configMock.get("user.signin.type.default", "Anonymous")).toReturn("Anonymous");
-        stub(configMock.get("user.login.type.default", "NA")).toReturn("NA");
-
-        UserDataCache userCacheMock = new UserDataCache(configMock,jobMetrics, redisConnectMock);
-        deNormalizationTask = new DeNormalizationTask(configMock, contextMock, userCacheMock , contentCacheMock, dailcodeCacheMock, jobMetrics, redisConnectMock);
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicIfDidIsNullWithUserContentEmptyData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), false);
-                assertEquals(flags.get("content_data_retrieved"), false);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicIfDidIsNullWithUserContentData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID);
-        Map<String, Object> content = new HashMap<>();
-        content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook");
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(content);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 5);
-                Map<String, Object> contentData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType);
-                assertEquals(contentData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), true);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithUserContentData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}");
-        Map<String, Object> content = new HashMap<>();
-        content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook");
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(content);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 5);
-                Map<String, Object> contentData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType);
-                assertEquals(contentData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), true);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithoutRollUpID() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                System.out.println(outputMessage);
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 5);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), false);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithRollUpIdInCollectionData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECT_ROLLUP);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}");
-        Map<String, Object> collection = new HashMap<>();
-        collection.put("name", "collection-1");
-        collection.put("objecttype", "TextBook");
-        collection.put("contenttype", "TextBook");
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(collection);
-        deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                System.out.println();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> collectionData = new Gson().fromJson(outputEvent.get("collectiondata").toString(), mapType);
-                assertEquals(collectionData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("collection_data_retrieved"), true);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithObjectIdInCollectionData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECT_ROLLUP);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}");
-        Map<String, Object> collection = new HashMap<>();
-        collection.put("name", "collection-1");
-        collection.put("objecttype", "TextBook");
-        collection.put("contenttype", "TextBook");
-        stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(collection);
-        deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> collectionData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType);
-                assertEquals(collectionData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("collection_data_retrieved"), false);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithObjectIdEqualsRollUpIDInCollectionData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_OBJECTID_EQUALS_ROLLUPID);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"state\":\"Karnataka\"}");
-        Map<String, Object> collection = new HashMap<>();
-        collection.put("name", "collection-1");
-        collection.put("objecttype", "TextBook");
-        collection.put("contenttype", "TextBook");
-        Map<String, Object> content = new HashMap<>();
-        content.put("name", "content-1"); content.put("objecttype", "Content"); content.put("contenttype", "TextBook");
-        stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(collection);
-        stub(contentCacheMock.getData("do_31277438304183091217888")).toReturn(content);
-        deNormalizationTask.process(envelopeMock,collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                System.out.println(outputEvent);
-                Map<String, Object> collectionData = new Gson().fromJson(outputEvent.get("contentdata").toString(), mapType);
-                assertEquals(collectionData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("content_data_retrieved"), true);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicIfContentIdIsNullWithUserData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_OBJECT);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"type\":\"Registered\"}");
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals("3.0", outputEvent.get("ver").toString());
-                assertNull(outputEvent.get("contentdata"));
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(true, flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicIfContentDataIsEmptyWithUserAsSystem() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"type\":\"Registered\"}");
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals("3.0", outputEvent.get("ver").toString());
-                assertNull(outputEvent.get("contentdata"));
-                assertNull(outputEvent.get("userdata"));
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertNull(flags.get("user_data_retrieved"));
-                assertEquals(false, flags.get("content_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    public void shouldSendEventsToSuccessTopicWithStringDialCodeData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_DIALCODE_AS_STRING);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        List ids = new ArrayList(); ids.add("8ZEDTP");
-        Map dataMap = new HashMap(); dataMap.put("identifier", "8ZEDTP"); dataMap.put("channel", "test-channel");
-        dataMap.put("status", "Draft");
-        List<Map> data = new ArrayList<>(); data.add(dataMap);
-        stub(dailcodeCacheMock.getData(ids)).toReturn(data);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                List<Map<String, Object>> dialcodesList = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), List.class);
-                assertEquals(dialcodesList.size(), 1);
-                Map data = dialcodesList.get(0);
-                assertEquals(data.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertNull(flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertEquals(true, flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    public void shouldSendEventsToSuccessTopicWithListDialCodeData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_DIALCODE_AS_LIST);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        List ids = new ArrayList(); ids.add("8ZEDTP"); ids.add("4ZEDTP");
-        Map dataMap1 = new HashMap(); dataMap1.put("identifier", "8ZEDTP"); dataMap1.put("channel", "test-channel");
-        dataMap1.put("status", "Draft");
-        Map dataMap2 = new HashMap(); dataMap2.put("identifier", "4ZEDTP"); dataMap2.put("channel", "test-channel");
-        dataMap2.put("status", "Draft");
-        List<Map> data = new ArrayList<>(); data.add(dataMap1); data.add(dataMap2);
-        stub(dailcodeCacheMock.getData(ids)).toReturn(data);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>(){}.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                assertNull(outputEvent.get("userdata"));
-                List<Map<String, Object>> dialcodesList = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), List.class);
-                assertEquals(dialcodesList.size(), 2);
-                Map data1 = dialcodesList.get(0);
-                assertEquals(data1.size(), 3);
-                Map data2 = dialcodesList.get(1);
-                assertEquals(data2.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertNull(flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertEquals(true, flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithOutDialCodeData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITHOUT_DIALCODE);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                assertNull(outputEvent.get("userdata"));
-                assertNull(outputEvent.get("dialcodedata"));
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertNull(flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertNull(flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithOutAnyDenormData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_DEVICEDATA);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), false);
-                assertEquals(flags.get("content_data_retrieved"), false);
-                return true;
-            }
-        }));
-    }
-
-    public void shouldSendEventsToSuccessTopicWithExistingEmptyStateCode() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITH_EMPTY_LOC);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), false);
-                assertEquals(flags.get("content_data_retrieved"), false);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithDialCodeLowerCaseDataByObjectLookup() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_AS_OBJECT);
-        stub(contentCacheMock.getData("977D3I")).toReturn(null);
-        Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel");
-        dataMap.put("status", "Draft");
-        stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class);
-                assertEquals(data.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(false, flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertEquals(true, flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    //@Test
-    public void shouldSendEventsToSuccessTopicWithStringDialCodeDataCamelCase() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_CAMELCASE_DIALCODE_AS_STRING);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        List ids = new ArrayList(); ids.add("8ZEDTP");
-        Map dataMap = new HashMap(); dataMap.put("identifier", "8ZEDTP"); dataMap.put("channel", "test-channel");
-        dataMap.put("status", "Draft");
-        List<Map> data = new ArrayList<>(); data.add(dataMap);
-        stub(dailcodeCacheMock.getData(ids)).toReturn(data);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertEquals(outputMessage.contains("dialcodes"), true);
-                assertEquals(outputMessage.contains("dialCodes"), false);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithAlteredEts() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_FUTURE_ETS);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "2.1");
-                Long ets = new Gson().fromJson(outputEvent.get("ets").toString(), Long.class);
-                assertFalse(2530937155000L == ets);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfEventIsOlder() throws Exception{
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_OLDER_ETS);
-        stub(contentCacheMock.getData("do_31249561779090227216256")).toReturn(null);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    public void shouldSendSummaryEventsToSuccessTopicWithUserData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.WFS_EVENT);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "1.1");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 5);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), null);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSkipOtherSummaryEvent() throws Exception{
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_SUMMARY_EVENT);
-        Answer answer = new Answer();
-        doAnswer(answer).when(jobMetrics).incSkippedCounter();
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        assertEquals(true, answer.isSkipped);
-    }
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicForLogEvents() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT);
-        jedisMock.set("0b251080-3230-415e-a593-ab7c1fac7ae3","{\"grade\":[4,5],\"type\":\"Registered\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(true, flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldPushToSuccessTopicForErrorEvents() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT);
-        Answer answer = new Answer();
-        doAnswer(answer).when(jobMetrics).incSuccessCounter();
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        assertEquals(true, answer.isSuccess);
-    }
-
-    class Answer implements org.mockito.stubbing.Answer {
-
-        private Boolean isSkipped = false;
-        private Boolean isSuccess = false;
-        @Override
-        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
-            isSkipped = true;
-            isSuccess = true;
-            return null;
-        }
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithQRCodeDataByObjectLookup() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_QR_AS_OBJECT);
-        stub(contentCacheMock.getData("977D3I")).toReturn(null);
-        Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel");
-        dataMap.put("status", "Draft");
-        stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                assertNull(outputEvent.get("userdata"));
-                Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class);
-                assertEquals(data.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(false, flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertEquals(true, flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventsToSuccessTopicWithDialCodeCamelCaseDataByObjectLookup() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_CAMELCASE_AS_OBJECT);
-        stub(contentCacheMock.getData("977D3I")).toReturn(null);
-        Map dataMap = new HashMap(); dataMap.put("identifier", "977D3I"); dataMap.put("channel", "test-channel");
-        dataMap.put("status", "Draft");
-        stub(dailcodeCacheMock.getData("977D3I")).toReturn(dataMap);
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                assertNull(outputEvent.get("contentdata"));
-                assertNull(outputEvent.get("userdata"));
-                Map data = new Gson().fromJson(outputEvent.get("dialcodedata").toString(), Map.class);
-                assertEquals(data.size(), 3);
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(false, flags.get("user_data_retrieved"));
-                assertNull(flags.get("content_data_retrieved"));
-                assertEquals(true, flags.get("dialcode_data_retrieved"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendAUDITEventsToSuccessTopicWithUserData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT);
-        jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                String jedisMockData = jedisMock.get("393407b1-66b1-4c86-9080-b2bce9842886");
-                assertEquals(userData.size(), 5);
-                assert jedisMockData.contains("grade");
-                assert jedisMockData.contains("district");
-                assert jedisMockData.contains("type");
-                assert jedisMockData.contains("state");
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), null);
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldStampUserSigninAndLoginTypeForAllEvents() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT);
-        String user_id="e9d73b7b-211a-43f4-8c01-cc9333757a9d";
-        jedisMock.set(user_id,"{\"subject\":[],\"grade\":[],\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"student\"}");
-        deNormalizationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                assertEquals(outputEvent.get("ver").toString(), "3.0");
-                Map<String, Object> userData = new Gson().fromJson(outputEvent.get("userdata").toString(), mapType);
-                assertEquals(userData.size(), 4);
-                assertEquals(userData.get("usersignintype"),"Self-Signed-In");
-                Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-                assertEquals(flags.get("user_data_retrieved"), true);
-                assertEquals(flags.get("content_data_retrieved"), null);
-                return true;
-            }
-        }));
-    }
-}
diff --git a/data-pipeline/deploy.sh b/data-pipeline/deploy.sh
deleted file mode 100755
index 71d833e6b1..0000000000
--- a/data-pipeline/deploy.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-mvn clean package
-rm -rf deploy/samza
-mkdir -p deploy/samza
-tar -xvf ./target/ekstep-samza-0.0.1-dist.tar.gz -C deploy/samza
-deploy/samza/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file://$PWD/deploy/samza/config/reverse-search.properties
\ No newline at end of file
diff --git a/data-pipeline/derived-de-duplication/pom.xml b/data-pipeline/derived-de-duplication/pom.xml
deleted file mode 100644
index 0492d41779..0000000000
--- a/data-pipeline/derived-de-duplication/pom.xml
+++ /dev/null
@@ -1,179 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>derived-de-duplication</artifactId>
-    <version>0.0.3</version>
-    <packaging>jar</packaging>
-    <name>DerivedDeduplication</name>
-    <description>
-       Identifying Derived Duplicate Events
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/derived-de-duplication/src/main/assembly/src.xml b/data-pipeline/derived-de-duplication/src/main/assembly/src.xml
deleted file mode 100644
index a14eb7749d..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/derived-de-duplication.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:derived-de-duplication</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties b/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties
deleted file mode 100644
index 4bf2eb9d0c..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/config/derived-de-duplication.properties
+++ /dev/null
@@ -1,87 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.DerivedDeDuplication
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_derived_de_duplication_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.DeDuplicationTask
-task.inputs=kafka.__env__.telemetry.derived
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# YARN container configuration
-# yarn.am.container.memory.mb=__yarn_am_container_mb__
-# yarn.container.memory.mb=__dedup_job_yarn_container_memory_mb__
-
-output.success.topic.name=__env__.telemetry.derived.unique
-output.failed.topic.name=__env__.telemetry.failed
-output.duplicate.topic.name=__env__.telemetry.duplicate
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,duplicate-event-count
-
-# redis
-#redis.host=localhost
-redis.host=__redis_host__
-#redis.port=6379
-redis.port=__redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.duplicationstore.id=11
-redis.database.key.expiry.seconds=86400
\ No newline at end of file
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 69c2e4a7af..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-
-import org.apache.commons.lang.StringUtils;
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-
-    public void markSkipped() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.derived_dd_processed", false);
-        telemetry.add("flags.derived_dd_checksum_present", false);
-    }
-
-    public void markDuplicate() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.derived_dd_processed", false);
-        telemetry.add("flags.derived_dd_duplicate_event", true);
-    }
-
-    public void markSuccess() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.derived_dd_processed", true);
-        telemetry.add("type", "events");
-    }
-
-    public void markRedisFailure() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.derived_dd_processed", false);
-        telemetry.add("flags.derived_dd_redis_failure", true);
-        telemetry.add("type", "events");
-    }
-
-    public void markFailure(String error, DeDuplicationConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.derived_dd_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("metadata.derived_dd_error", error);
-        telemetry.add("metadata.src", config.jobName());
-    }
-
-}
-
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java
deleted file mode 100644
index c95f63525d..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/service/DeDuplicationService.java
+++ /dev/null
@@ -1,67 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import org.ekstep.ep.samza.task.DeDuplicationSink;
-import org.ekstep.ep.samza.task.DeDuplicationSource;
-import org.ekstep.ep.samza.util.DeDupEngine;
-
-import redis.clients.jedis.exceptions.JedisException;
-
-public class DeDuplicationService {
-	private static Logger LOGGER = new Logger(DeDuplicationService.class);
-	private final DeDupEngine deDupEngine;
-	private final DeDuplicationConfig config;
-
-
-	public DeDuplicationService(DeDupEngine deDupEngine, DeDuplicationConfig config) {
-		this.deDupEngine = deDupEngine;
-		this.config = config;
-	}
-
-	public void process(DeDuplicationSource source, DeDuplicationSink sink) throws Exception {
-		Event event = null;
-
-		try {
-			event = source.getEvent();
-			String checksum = event.getChecksum();
-			if (checksum == null) {
-				LOGGER.info(event.id(), "EVENT WITHOUT CHECKSUM & MID, PASSING THROUGH : {}", event);
-				event.markSkipped();
-				sink.toSuccessTopic(event);
-				return;
-			}
-
-			if (isDupCheckRequired(event)) {
-				if (!deDupEngine.isUniqueEvent(checksum)) {
-					LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum);
-					event.markDuplicate();
-					sink.toDuplicateTopic(event);
-					return;
-				}
-
-				LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE");
-
-				deDupEngine.storeChecksum(checksum);
-			}
-			event.markSuccess();
-			sink.toSuccessTopic(event);
-
-		} catch (JedisException e) {
-			LOGGER.error(null, "Exception when retrieving data from redis:  ", e);
-			event.markRedisFailure();
-			sink.toSuccessTopicIfRedisException(event);
-			throw e;
-		} catch (JsonSyntaxException e) {
-			LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-			sink.toMalformedEventsTopic(source.getMessage());
-		}
-	}
-
-	public boolean isDupCheckRequired(Event event) {
-		return (config.inclusiveProducerIds().isEmpty() || (null != event.producerId() && config.inclusiveProducerIds().contains(event.producerId())));
-	}
-}
-
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java
deleted file mode 100644
index ea3660302e..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationConfig.java
+++ /dev/null
@@ -1,68 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class DeDuplicationConfig {
-
-    private final String JOB_NAME = "DerivedDeDuplication";
-    private String successTopic;
-    private String failedTopic;
-    private String duplicateTopic;
-    private String malformedTopic;
-    private final int dupStore;
-    private int expirySeconds;
-    private List<String> includedProducerIds;
-
-
-    public DeDuplicationConfig(Config config) {
-        successTopic = config.get("output.success.topic.name", "telemetry.derived.unique");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        dupStore = config.getInt("redis.database.duplicationstore.id", 11);
-        expirySeconds = config.getInt("redis.database.key.expiry.seconds", 86400);
-        if (!config.get("dedup.producer.include.ids", "").isEmpty()) {
-            includedProducerIds = config.getList("dedup.producer.include.ids", new ArrayList<>());
-        } else {
-            includedProducerIds = new ArrayList<>();
-        }
-
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String duplicateTopic() {
-        return duplicateTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public int dupStore() {
-        return dupStore;
-    }
-
-    public int expirySeconds() {
-        return expirySeconds;
-    }
-
-    public List<String> inclusiveProducerIds() {
-        return includedProducerIds;
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java
deleted file mode 100644
index a100e4be50..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSink.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class DeDuplicationSink extends BaseSink {
-
-    private DeDuplicationConfig config;
-
-    public DeDuplicationSink(MessageCollector collector, JobMetrics metrics, DeDuplicationConfig config) {
-        super(collector, metrics);
-        this.config = config;
-    }
-
-    public void toSuccessTopic(Event event) {
-        toTopic(config.successTopic(), event.did(), event.getJson());
-        metrics.incSuccessCounter();
-    }
-
-    public void toSuccessTopicIfRedisException(Event event) {
-        toTopic(config.successTopic(), event.did(), event.getJson());
-        metrics.incCacheErrorCounter();
-    }
-
-    public void toDuplicateTopic(Event event) {
-        toTopic(config.duplicateTopic(), event.did(), event.getJson());
-        metrics.incDuplicateCounter();
-    }
-
-    public void toMalformedEventsTopic(String message) {
-        toTopic(config.malformedTopic(), null, message);
-        metrics.incFailedCounter();
-    }
-
-    public void toErrorTopic(Event event) {
-        toTopic(config.failedTopic(), event.did(), event.getJson());
-        metrics.incErrorCounter();
-    }
-
-}
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java
deleted file mode 100644
index 0b8fceec3b..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationSource.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class DeDuplicationSource {
-	private static Logger LOGGER = new Logger(DeDuplicationSource.class);
-
-	private IncomingMessageEnvelope envelope;
-
-	public DeDuplicationSource(IncomingMessageEnvelope envelope) {
-		this.envelope = envelope;
-	}
-
-	public Event getEvent() {
-		String message = (String) envelope.getMessage();
-		@SuppressWarnings("unchecked")
-		Map<String, Object> jsonMap = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-		return new Event(jsonMap);
-	}
-
-	public String getMessage() {
-		return envelope.toString();
-	}
-	public SystemStreamPartition getSystemStreamPartition() { return envelope.getSystemStreamPartition();}
-	public String getOffset() { return envelope.getOffset();}
-}
diff --git a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java b/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java
deleted file mode 100644
index 529ca88107..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/java/org/ekstep/ep/samza/task/DeDuplicationTask.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.DeDuplicationService;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class DeDuplicationTask extends BaseSamzaTask {
-
-	private DeDuplicationConfig config;
-	private JobMetrics metrics;
-	private DeDuplicationService service;
-
-	public DeDuplicationTask(Config config, TaskContext context,
-							 DeDupEngine deDupEngine) {
-		init(config, context, deDupEngine);
-	}
-
-	public DeDuplicationTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		init(config, context, null);
-	}
-
-	private void init(Config config, TaskContext context,
-					  DeDupEngine deDupEngine) {
-		this.config = new DeDuplicationConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		deDupEngine = deDupEngine == null ?
-				new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine;
-		service = new DeDuplicationService(deDupEngine, this.config);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) throws Exception {
-		DeDuplicationSource source = new DeDuplicationSource(envelope);
-		DeDuplicationSink sink = new DeDuplicationSink(collector, metrics, config);
-		service.process(source, sink);
-	}
-	
-}
diff --git a/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml b/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/derived-de-duplication/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
deleted file mode 100644
index 9e88c40ae7..0000000000
--- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import org.junit.Assert;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import java.util.Map;
-
-public class EventTest {
-
-    private DeDuplicationConfig configMock;
-
-    @Test
-    public void shouldAddSkipFlags() {
-        String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-        Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-        event.markSkipped();
-        Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-        Assert.assertNotNull(flagData);
-        Assert.assertEquals(flagData.get("derived_dd_checksum_present"), false);
-        Assert.assertEquals(flagData.get("derived_dd_processed"), false);
-
-    }
-
-    @Test
-    public void shouldMarkTheRedisFailure() {
-        String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-        Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-        event.markRedisFailure();
-        Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-        Assert.assertNotNull(flagData);
-        Assert.assertEquals(flagData.get("derived_dd_redis_failure"), true);
-        Assert.assertEquals(flagData.get("derived_dd_processed"), false);
-
-    }
-
-    @Test
-    public void shouldMarkFailure() {
-        configMock = Mockito.mock(DeDuplicationConfig.class);
-        String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-        Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-        event.markFailure("Invalid Event", configMock);
-        Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-        Assert.assertNotNull(flagData);
-        Assert.assertEquals(flagData.get("derived_dd_processed"), false);
-        Object metaData = event.getMap().get("metadata");
-        Assert.assertNotNull(metaData);
-
-    }
-
-}
diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index ce61e49754..0000000000
--- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,166 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.util.Map;
-
-public class EventFixture {
-    private static final String EVENT_WITH_CHECKSUM = "{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"9a78dacddfcdfae7a9679b143e348158\"\n" +
-            "  },\n" +
-            "  \"eid\": \"INTERACT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"id\": \"do_312764599773110272147\",\n" +
-            "    \"type\": \"TOUCH\",\n" +
-            "    \"pageid\": \"ContentApp-EndScreen\",\n" +
-            "    \"subtype\": \"ContentID\"\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561620619274,\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"ver\": \"1.14.5\",\n" +
-            "      \"pid\": \"sunbird-portal.contentplayer\",\n" +
-            "      \"id\": \"dev.diksha.portal\"\n" +
-            "    },\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"env\": \"contentplayer\",\n" +
-            "    \"did\": \"9a78dacddfcdfae7a9679b143e348158\",\n" +
-            "    \"sid\": \"8c416b30-4e47-a695-b1d2-9cd030eadeff\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"DialCode\",\n" +
-            "        \"id\": \"DNILBR\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"ContentSession\",\n" +
-            "        \"id\": \"88de9e1dfa6f237c076278a02cc4b305\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"INTERACT:fd63ff6bd0eedf05cf57c0058588ae69\",\n" +
-            "  \"object\": {\n" +
-            "    \"ver\": \"1\",\n" +
-            "    \"id\": \"do_312764599773110272147\",\n" +
-            "    \"type\": \"Content\",\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"do_31276594374681395211909\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"syncts\": 1561620619795,\n" +
-            "  \"@timestamp\": \"2019-06-27T07:30:19.795Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    private static final String EVENT_WITH_MID = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\",\n" +
-            "    \"mid\": \"22e1430f2e5f339230dbf9595b060008\"" +
-            "}";
-
-    private static final String EVENT_WITHOUT_CHECKSUM = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\"" +
-            "}";
-
-    private static final String EVENT_WITH_EMTPY_CHANNEL = "{\n" +
-            "    \"edata\": {\n" +
-            "        \"eks\": {\n" +
-            "            \"loc\": \"\",\n" +
-            "            \"uid\": \"6\",\n" +
-            "            \"age\": 5,\n" +
-            "            \"handle\": \"Jijesh\",\n" +
-            "            \"standard\": -1,\n" +
-            "            \"language\": \"ML\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"eid\": \"GE_CREATE_PROFILE\",\n" +
-            "    \"did\": \"cbeda6a2ef327eaee21008de6495f89476aba58d\",\n" +
-            "    \"gdata\": {\n" +
-            "        \"id\": \"genieservice.android\",\n" +
-            "        \"ver\": \"1.0.local-qa-debug\"\n" +
-            "    },\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"context\": {\n" +
-            "    \"channel\": \"\"\n" +
-            "    },\n" +
-            "    \"tags\": [],\n" +
-            "    \"ts\": \"2015-09-27T13:03:43-04:00\",\n" +
-            "    \"uid\": \"6\",\n" +
-            "    \"ver\": \"1.0\"" +
-            "}";
-
-    public static Map<String, Object> EventWithChecksumMap() {
-        return new Gson().fromJson(EVENT_WITH_CHECKSUM, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithMidMap() {
-        return new Gson().fromJson(EVENT_WITH_MID, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithoutChecksumFieldMap() {
-        return new Gson().fromJson(EVENT_WITHOUT_CHECKSUM, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static Map<String, Object> EventWithEmptyChannel() {
-        return new Gson().fromJson(EVENT_WITH_EMTPY_CHANNEL, new TypeToken<Map<String, Object>>() {
-        }.getType());
-    }
-
-    public static String EventWithChecksumJson() {
-        return EVENT_WITH_CHECKSUM;
-    }
-
-}
-
diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java
deleted file mode 100644
index 185dc5d6b6..0000000000
--- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/system/EventTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.task.DeDuplicationConfig;
-import org.junit.Assert;
-import org.junit.Test;
-import java.util.Map;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class EventTest {
-    @Test
-    public void shouldReturnChecksumIfPresent(){
-
-        Event event = new Event(EventFixture.EventWithChecksumMap());
-        Assert.assertEquals("INTERACT:fd63ff6bd0eedf05cf57c0058588ae69", (String) event.getChecksum());
-    }
-
-    @Test
-    public void shouldReturnMidIfPresent(){
-
-        Event event = new Event(EventFixture.EventWithMidMap());
-        Assert.assertEquals("22e1430f2e5f339230dbf9595b060008", (String) event.getChecksum());
-    }
-
-    @Test
-    public void shouldReturnNullIfChecksumAndMidAreAbsent(){
-
-        Event event = new Event(EventFixture.EventWithoutChecksumFieldMap());
-        Assert.assertNull(event.getChecksum());
-    }
-}
-
diff --git a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java b/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java
deleted file mode 100644
index 6663ed8916..0000000000
--- a/data-pipeline/derived-de-duplication/src/test/java/org/ekstep/ep/samza/task/DeDuplicationTaskTest.java
+++ /dev/null
@@ -1,141 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-import redis.clients.jedis.Jedis;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class DeDuplicationTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.derived.unique";
-    private static final String FAILED_TOPIC = "telemetry.unique.fail";
-    private static final String DUPLICATE_TOPIC = "telemetry.duplicate";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private DeDuplicationTask deDuplicationTask;
-    private DeDupEngine deDupEngineMock;
-    private Jedis jedisMock = new MockJedis("duplicationtest");
-    private int dupStoreId = 1;
-
-    @Before
-    public void setUp() {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = Mockito.mock(TaskContext.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = Mockito.mock(Config.class);
-        RedisConnect redisConnectMock = mock(RedisConnect.class);
-        deDupEngineMock = mock(DeDupEngine.class);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC);
-        stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-        stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC);
-        stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-        stub(metricsRegistry.newCounter(anyString(), anyString()))
-                .toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(0)));
-        stub(configMock.getInt("redis.database.duplicationstore.id", dupStoreId)).toReturn(dupStoreId);
-        when(configMock.get("dedup.producer.include.ids", "")).thenReturn("dev.diskha.portal");
-        stub(configMock.getList("dedup.producer.include.ids", new ArrayList<>())).toReturn(Arrays.asList("dev.diksha.portal"));
-
-
-        deDuplicationTask = new DeDuplicationTask(configMock, contextMock, deDupEngineMock);
-    }
-
-    @Test
-    public void ShouldSendEventToSuccessTopicIfEventIsUnique() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson());
-
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    @Test
-    public void ShouldSendEventsToDuplicateTopicIfEventIsDuplicate() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC)));
-    }
-
-    @Test
-    public void ShouldSendEventsToMalformedTopicIfEventIsMalformed() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn("{'metadata':{'checksum':'sajksajska'}");
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-    }
-
-    @Test(expected = Exception.class)
-    public void ShouldSendEventsToFailedTopicForAnyUnhandledException() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenThrow(new Exception());
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    public void ShouldStoreChecksumIfEventIsUnique() throws Exception {
-
-        when(envelopeMock.getMessage()).thenReturn(EventFixture.EventWithChecksumJson());
-        when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-
-        deDuplicationTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(deDupEngineMock, times(1)).isUniqueEvent(anyString());
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-}
diff --git a/data-pipeline/device-profile-updater/pom.xml b/data-pipeline/device-profile-updater/pom.xml
deleted file mode 100644
index e0fb3eee0e..0000000000
--- a/data-pipeline/device-profile-updater/pom.xml
+++ /dev/null
@@ -1,185 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>jobs</artifactId>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <version>0.0.1</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>device-profile-updater</artifactId>
-    <version>0.1.8</version>
-    <name>DeviceProfileUpdater</name>
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.opentable.components</groupId>
-            <artifactId>otj-pg-embedded</artifactId>
-            <version>0.13.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
-            <version>RELEASE</version>
-            <scope>compile</scope>
-        </dependency>
-        <dependency>
-            <groupId>it.ozimov</groupId>
-            <artifactId>embedded-redis</artifactId>
-            <version>0.7.1</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-
-</project>
\ No newline at end of file
diff --git a/data-pipeline/device-profile-updater/src/main/assembly/src.xml b/data-pipeline/device-profile-updater/src/main/assembly/src.xml
deleted file mode 100644
index 051f966cc6..0000000000
--- a/data-pipeline/device-profile-updater/src/main/assembly/src.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/device-profile-updater.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:device-profile-updater</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-                <include>redis.clients:2.9.0</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties b/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties
deleted file mode 100644
index 5926a24dd0..0000000000
--- a/data-pipeline/device-profile-updater/src/main/config/device-profile-updater.properties
+++ /dev/null
@@ -1,91 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.DeviceProfileUpdater
-
-# YARN
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__device_profile_updater_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.DeviceProfileUpdaterTask
-task.inputs=kafka.__env__.events.deviceprofile
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-#systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__ingestion_zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-#systems.kafka.producer.bootstrap.servers=localhost:9092
-systems.kafka.producer.bootstrap.servers=__ingestion_kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-output.metrics.topic.name=__env__.pipeline_metrics
-output.malformed.topic.name=__env__.telemetry.malformed
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,failed-message-count,error-message-count,device-db-update-count,device-cache-update-count
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# postgres
-postgres.host=__postgres_host__
-#postgres.host=localhost
-postgres.port=__postgres_port__
-#postgres.port=5432
-postgres.user=__postgres_user__
-sensitive.postgres.password=__postgres_password__
-postgres.db=__postgres_db__
-postgres.device_profile_table=__env___device_profile
-postgres.maxConnections = 2
-
-# redis
-#redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.deviceStore.id=2
-
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java
deleted file mode 100644
index b6f35efa2a..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-
-public class DeviceProfile {
-  private String countryCode;
-  private String country;
-  private String stateCode;
-  private String state;
-  private String city;
-  private String districtCustom;
-  private String stateCodeCustom;
-  private String stateCustomName;
-  private String userDeclaredState;
-  private String userDeclaredDistrict;
-  private Map<String, String> uaspec;
-  private Map<String, String> devicespec;
-  private Long firstAccess;
-  private Long user_declared_on;
-  private Long api_last_updated_on;
-  private Gson gson = new Gson();
-  private Type type = new TypeToken<Map<String, Object>>() {}.getType();
-
-  public DeviceProfile() {
-    this.countryCode = "";
-    this.country = "";
-    this.stateCode = "";
-    this.state = "";
-    this.city = "";
-    this.districtCustom = "";
-    this.stateCodeCustom = "";
-    this.stateCustomName = "";
-    this.userDeclaredState = "";
-    this.userDeclaredDistrict = "";
-    this.uaspec = new HashMap<>();
-    this.devicespec = new HashMap<>();
-    this.firstAccess = 0L;
-    this.user_declared_on = 0L;
-    this.api_last_updated_on = 0L;
-  }
-
-  public Map<String, String> toMap() {
-    Map<String, String> values = new HashMap<>();
-    values.put("country_code", DeviceProfile.getValueOrDefault(this.countryCode, ""));
-    values.put("country", DeviceProfile.getValueOrDefault(this.country, ""));
-    values.put("state_code", DeviceProfile.getValueOrDefault(this.stateCode, ""));
-    values.put("state", DeviceProfile.getValueOrDefault(this.state, ""));
-    values.put("city", DeviceProfile.getValueOrDefault(this.city, ""));
-    values.put("district_custom", DeviceProfile.getValueOrDefault(this.districtCustom, ""));
-    values.put("state_custom", DeviceProfile.getValueOrDefault(this.stateCustomName, ""));
-    values.put("state_code_custom", DeviceProfile.getValueOrDefault(this.stateCodeCustom, ""));
-    values.put("user_declared_state", DeviceProfile.getValueOrDefault(this.userDeclaredState, ""));
-    values.put("user_declared_district", DeviceProfile.getValueOrDefault(this.userDeclaredDistrict, ""));
-    values.put("uaspec", gson.toJson(DeviceProfile.getValueOrDefault(this.uaspec, new HashMap<>())));
-    values.put("devicespec", gson.toJson(DeviceProfile.getValueOrDefault(this.devicespec, new HashMap<>())));
-    values.put("firstaccess", DeviceProfile.getValueOrDefault(String.valueOf(this.firstAccess), ""));
-    values.put("user_declared_on", DeviceProfile.getValueOrDefault(String.valueOf(this.user_declared_on), ""));
-    values.put("api_last_updated_on", DeviceProfile.getValueOrDefault(String.valueOf(this.api_last_updated_on), ""));
-    return values;
-  }
-
-  public DeviceProfile fromMap(Map<String, String> map) {
-    this.countryCode = map.getOrDefault("country_code", "");
-    this.country = map.getOrDefault("country", "");
-    this.stateCode = map.getOrDefault("state_code", "");
-    this.state = map.getOrDefault("state", "");
-    this.city = map.getOrDefault("city", "");
-    this.districtCustom = map.getOrDefault("district_custom", "");
-    this.stateCustomName = map.getOrDefault("state_custom", "");
-    this.stateCodeCustom = map.getOrDefault("state_code_custom", "");
-    this.userDeclaredState = map.getOrDefault("user_declared_state", "");
-    this.userDeclaredDistrict = map.getOrDefault("user_declared_district", "");
-    this.uaspec = gson.fromJson(map.getOrDefault("uaspec", ""), type);
-    this.devicespec = gson.fromJson(map.getOrDefault("device_spec", ""), type);
-    this.firstAccess = Long.valueOf(map.getOrDefault("first_access", "0"));
-    this.user_declared_on = Long.valueOf(map.getOrDefault("api_last_updated_on", "0"));
-    this.api_last_updated_on = Long.valueOf(map.getOrDefault("api_last_updated_on", "0"));
-    return this;
-  }
-
-  public static <T> T getValueOrDefault(T value, T defaultValue) {
-    return value == null ? defaultValue : value;
-  }
-}
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java
deleted file mode 100644
index 083541abf6..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/service/DeviceProfileUpdaterService.java
+++ /dev/null
@@ -1,182 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.google.gson.JsonSyntaxException;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import org.ekstep.ep.samza.task.DeviceProfileUpdaterSink;
-import org.ekstep.ep.samza.task.DeviceProfileUpdaterSource;
-import org.ekstep.ep.samza.util.PostgresConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.postgresql.util.PGobject;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-import java.lang.reflect.Type;
-import com.google.gson.reflect.TypeToken;
-
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Timestamp;
-import java.util.*;
-import java.util.stream.Collectors;
-
-public class DeviceProfileUpdaterService {
-
-    private static Logger LOGGER = new Logger(DeviceProfileUpdaterService.class);
-    private RedisConnect redisConnect;
-    private Jedis deviceStoreConnection;
-    private int deviceStoreDb;
-    private PostgresConnect postgresConnect;
-    private String postgres_table;
-    private Gson gson = new Gson();
-    private Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-
-    public DeviceProfileUpdaterService(Config config, RedisConnect redisConnect, PostgresConnect postgresConnect) {
-        this.redisConnect = redisConnect;
-        this.postgresConnect = postgresConnect;
-        this.deviceStoreDb = config.getInt("redis.database.deviceStore.id", 2);
-        this.deviceStoreConnection = redisConnect.getConnection(deviceStoreDb);
-        this.postgres_table = config.get("postgres.device_profile_table", "device_profile");
-    }
-
-    public void process(DeviceProfileUpdaterSource source, DeviceProfileUpdaterSink sink) throws Exception {
-        try {
-            Map<String, String> message = source.getMap();
-            updateDeviceDetails(message, sink);
-        } catch (JsonSyntaxException e) {
-            LOGGER.error(null, "INVALID EVENT: " + source.getMessage(), e);
-            sink.toMalformedTopic(source.getMessage());
-        }
-    }
-
-    private void updateDeviceDetails(Map<String, String> deviceData, DeviceProfileUpdaterSink sink) throws Exception {
-        if (deviceData.size() > 0) {
-
-            deviceData.values().removeAll(Collections.singleton(""));
-            deviceData.values().removeAll(Collections.singleton("{}"));
-
-            DeviceProfile deviceProfile = new DeviceProfile().fromMap(deviceData);
-            String deviceId = deviceData.get("device_id");
-            if (null != deviceId && !deviceId.isEmpty()) {
-
-                // Update device profile details in Postgres DB
-                addDeviceDataToDB(deviceId, deviceData);
-                sink.deviceDBUpdateSuccess();
-
-                // Update device profile details in Redis cache
-                addDeviceDataToCache(deviceId, deviceProfile);
-                sink.deviceCacheUpdateSuccess();
-
-                sink.success();
-                LOGGER.info(deviceId,"Updated successfully");
-            }
-            else { sink.failed(); }
-        }
-
-    }
-
-    private void addDeviceDataToCache(String deviceId, DeviceProfile deviceProfile) {
-        try {
-            addToCache(deviceId, deviceProfile, deviceStoreConnection);
-        } catch (JedisException ex) {
-            this.deviceStoreConnection.close();
-            this.deviceStoreConnection = redisConnect.getConnection(deviceStoreDb);
-            addToCache(deviceId, deviceProfile, deviceStoreConnection);
-        }
-    }
-
-    private void addDeviceDataToDB(String deviceId, Map<String, String> deviceData) throws Exception {
-        Long firstAccess = Long.parseLong(deviceData.get("first_access"));
-        //
-        Long lastUpdatedDate = Long.parseLong(deviceData.get("api_last_updated_on"));
-        List<String> parsedKeys = new ArrayList<>(Arrays.asList("first_access", "api_last_updated_on"));
-        deviceData.keySet().removeAll(parsedKeys);
-        String columns = formatValues(deviceData.keySet(),",");
-        String values = formatPrepareStatement(deviceData.values().size(),"?,");
-        String postgresQuery = String.format("INSERT INTO %s (api_last_updated_on,updated_date,%s) VALUES(?,?,%s?) ON CONFLICT(device_id) DO UPDATE SET (api_last_updated_on,updated_date,%s)=(?,?,%s?);",postgres_table, columns, values, columns, values);
-        PreparedStatement preparedStatement = postgresConnect.getConnection().prepareStatement(postgresQuery);
-
-        preparedStatement.setTimestamp(1, new Timestamp(lastUpdatedDate));  // Adding api_last_updated_on as timestamp to index 1 of preparestatement
-        preparedStatement.setTimestamp(deviceData.values().size()+3, new Timestamp(lastUpdatedDate));   // Adding api_last_updated_on as timestamp to 3rd index after the map size(for on conflict value)
-        preparedStatement.setTimestamp(2, new Timestamp(System.currentTimeMillis()));   // Adding updated_date as timestamp to index 2 of preparestatement
-        preparedStatement.setTimestamp(deviceData.values().size()+4, new Timestamp(System.currentTimeMillis()));    // Adding updated_date as timestamp to 4th index after the map size(for on conflict value)
-
-        setPrepareStatement(preparedStatement,2, deviceData);   // Adding map values to preparestatement from index after the api_last_updated_on and updated_on
-        setPrepareStatement(preparedStatement,deviceData.values().size()+4, deviceData);    // Adding map values from 4th index after map size as index(1-api_last_updated_on, 2-updated_on, 3-(size+2)map-values)
-
-        preparedStatement.executeUpdate();
-        preparedStatement.close();
-        String updateFirstAccessQuery = String.format("UPDATE %s SET first_access = '%s' WHERE device_id = '%s' AND first_access IS NULL",
-                postgres_table, new Timestamp(firstAccess).toString(), deviceId);
-        postgresConnect.execute(updateFirstAccessQuery);
-
-        if(null != deviceData.get("user_declared_state")) {
-            String updateUserDeclaredOnQuery = String.format("UPDATE %s SET user_declared_on = '%s' WHERE device_id = '%s' AND user_declared_on IS NULL",
-                    postgres_table, new Timestamp(lastUpdatedDate).toString(), deviceId);
-            postgresConnect.execute(updateUserDeclaredOnQuery);
-        }
-
-    }
-
-    private void addToCache(String deviceId, DeviceProfile deviceProfile, Jedis redisConnection) {
-        Map<String, String> deviceMap = deviceProfile.toMap();
-        deviceMap.values().removeAll(Collections.singleton(""));
-        deviceMap.values().removeAll(Collections.singleton("{}"));
-        if(deviceMap.get("user_declared_state") == null) {
-            deviceMap.remove("user_declared_on");
-        }
-        if (redisConnection.exists(deviceId)) {
-            Map<String, String> data = redisConnection.hgetAll(deviceId);
-            if(data.get("firstaccess") != null && !("0").equals(data.get("firstaccess"))) {
-                deviceMap.remove("firstaccess");
-            }
-            if(data.get("user_declared_on") != null && deviceMap.get("user_declared_on") != null) {
-                deviceMap.remove("user_declared_on");
-            }
-            redisConnection.hmset(deviceId, deviceMap);
-        } else {
-            redisConnection.hmset(deviceId, deviceMap);
-        }
-        LOGGER.debug(null, String.format("Device details for device id %s updated successfully", deviceId));
-    }
-
-    private String formatValues(Collection<?> values, String delimiter) {
-        return values.stream().map(Object::toString).collect(Collectors.joining(delimiter));
-    }
-
-    private String formatPrepareStatement(Integer length, String delimiter) {
-        return StringUtils.repeat(delimiter,length-1);
-    }
-
-    private void setPrepareStatement(PreparedStatement preparedStatement, Integer index, Map<String, String> deviceData) throws SQLException {
-        for (String value : deviceData.values()) {
-            index++;
-            PGobject jsonObject = new PGobject();
-            if(isJson(value)) {
-                jsonObject.setType("json");
-                jsonObject.setValue(gson.fromJson(value, JsonObject.class).toString());
-                preparedStatement.setObject(index, jsonObject);
-            }
-            else {
-                preparedStatement.setString(index, value);
-            }
-        }
-    }
-
-    public boolean isJson(String json) {
-        try {
-            gson.fromJson(json, Object.class);
-            Object jsonObjType = gson.fromJson(json, Object.class).getClass();
-            if(jsonObjType.equals(String.class) || jsonObjType.equals(Double.class)){
-                return false;
-            }
-            return true;
-        } catch (JsonSyntaxException ex) {
-            return false;
-        }
-    }
-
-}
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java
deleted file mode 100644
index 97bb2abe9d..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterConfig.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.ekstep.ep.samza.task;
-import org.apache.samza.config.Config;
-
-public class DeviceProfileUpdaterConfig {
-
-    private final String JOB_NAME = "DeviceProfileUpdater";
-    private String malformedTopic;
-
-    public DeviceProfileUpdaterConfig(Config config) {
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-}
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java
deleted file mode 100644
index f4403bf2c8..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSink.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-public class DeviceProfileUpdaterSink extends BaseSink {
-    private DeviceProfileUpdaterConfig config;
-
-    public DeviceProfileUpdaterSink(MessageCollector collector, JobMetrics metrics, DeviceProfileUpdaterConfig config) {
-        super(collector, metrics);
-        this.config = config;
-    }
-
-    public void deviceDBUpdateSuccess() { metrics.deviceDBUpdateSuccess(); }
-
-    public void deviceCacheUpdateSuccess() { metrics.deviceCacheUpdateSuccess(); }
-
-    public void success() {
-        metrics.incSuccessCounter();
-    }
-
-    public void failed() {
-        metrics.incFailedCounter();
-    }
-
-    public void toMalformedTopic(String message) {
-        toTopic(config.malformedTopic(), null, message);
-        metrics.incErrorCounter();
-    }
-}
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java
deleted file mode 100644
index 5c26ee45c2..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterSource.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-
-import java.lang.reflect.Type;
-import java.util.Map;
-
-public class DeviceProfileUpdaterSource {
-
-    private IncomingMessageEnvelope envelope;
-
-    public DeviceProfileUpdaterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    @SuppressWarnings("unchecked")
-    public Map<String, String> getMap() {
-        String message = (String) envelope.getMessage();
-        Type mapType = new TypeToken<Map<String, String>>(){}.getType();
-        return new Gson().fromJson(message, mapType);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-}
diff --git a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java b/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java
deleted file mode 100644
index efcc0f4d7b..0000000000
--- a/data-pipeline/device-profile-updater/src/main/java/org/ekstep/ep/samza/task/DeviceProfileUpdaterTask.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.DeviceProfileUpdaterService;
-import org.ekstep.ep.samza.util.PostgresConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class DeviceProfileUpdaterTask extends BaseSamzaTask {
-
-    private JobMetrics metrics;
-    private DeviceProfileUpdaterService service;
-    private DeviceProfileUpdaterConfig config;
-    private RedisConnect redisConnect;
-    private PostgresConnect postgresConnect;
-
-    public DeviceProfileUpdaterTask(Config config, TaskContext context) throws Exception { init(config, context); }
-
-    public DeviceProfileUpdaterTask() {
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) throws Exception{
-        redisConnect = new RedisConnect(config);
-        this.config = new DeviceProfileUpdaterConfig(config);
-        metrics = new JobMetrics(context, this.config.jobName());
-        postgresConnect = new PostgresConnect(config);
-        service = new DeviceProfileUpdaterService(config, redisConnect, postgresConnect);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelope);
-        DeviceProfileUpdaterSink sink = new DeviceProfileUpdaterSink(collector, metrics, config);
-        service.process(source, sink);
-    }
-    
-}
-
diff --git a/data-pipeline/device-profile-updater/src/main/resources/log4j.xml b/data-pipeline/device-profile-updater/src/main/resources/log4j.xml
deleted file mode 100644
index 7e0a688c9b..0000000000
--- a/data-pipeline/device-profile-updater/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-    <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-    <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-        <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-        <param name="append" value="true" />
-        <param name="maxFileSize" value="100MB" />
-        <param name="maxBackupIndex" value="5" />
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-        </layout>
-    </appender>
-    <root>
-        <priority value="info" />
-        <appender-ref ref="RollingAppender"/>
-        <appender-ref ref="jmx" />
-    </root>
-</log4j:configuration>
diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java
deleted file mode 100644
index 8060b0b421..0000000000
--- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/DeviceProfileServiceTest.java
+++ /dev/null
@@ -1,300 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import com.opentable.db.postgres.embedded.EmbeddedPostgres;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.service.Fixtures.EventFixture;
-import org.ekstep.ep.samza.task.DeviceProfileUpdaterTask;
-import org.ekstep.ep.samza.task.DeviceProfileUpdaterSource;
-import org.ekstep.ep.samza.task.DeviceProfileUpdaterSink;
-import org.ekstep.ep.samza.util.PostgresConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.JedisPool;
-import redis.clients.jedis.exceptions.JedisException;
-import redis.embedded.RedisServer;
-
-import java.lang.reflect.Type;
-import java.sql.*;
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotEquals;
-import static org.mockito.Mockito.*;
-
-public class DeviceProfileServiceTest {
-
-    private RedisConnect redisConnectMock;
-    private PostgresConnect postgresConnectMock;
-    RedisServer redisServer;
-    private IncomingMessageEnvelope envelopeMock;
-    private Jedis jedisMock;
-    private Connection connectionMock;
-    private Statement statementMock;
-    private Statement statement;
-    private Connection connection;
-    private DeviceProfileUpdaterService deviceProfileUpdaterService;
-    private DeviceProfileUpdaterSink deviceProfileUpdaterSinkMock;
-    private Config configMock;
-    private Integer deviceStoreId = 2;
-    private String postgres_table;
-    private Gson gson = new Gson();
-    private Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-
-    @Before
-    public void setUp() throws Exception {
-        postgresConnectMock=mock(PostgresConnect.class);
-
-        deviceProfileUpdaterSinkMock = mock(DeviceProfileUpdaterSink.class);
-        configMock = mock(Config.class);
-        connectionMock = mock(Connection.class);
-        statementMock = mock(Statement.class);
-        redisServer = new RedisServer(6379);
-        redisServer.start();
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        stub(configMock.getInt("redis.database.deviceStore.id", deviceStoreId)).toReturn(deviceStoreId);
-        stub(configMock.get("input.device.topic.name","device.profile")).toReturn("events.deviceprofile");
-        postgres_table = "device_profile";
-
-        EmbeddedPostgres pg = EmbeddedPostgres.start();
-        connection = pg.getPostgresDatabase().getConnection();
-        statement = connection.createStatement();
-
-        stub(connectionMock.createStatement()).toReturn(statement);
-        stub(postgresConnectMock.getConnection()).toReturn(connection);
-        stub(postgresConnectMock.resetConnection()).toReturn(connection);
-        statement.execute("CREATE TABLE device_profile(\n" +
-                "   device_id text PRIMARY KEY,\n" +
-                "   api_last_updated_on TIMESTAMP,\n" +
-                "    avg_ts float,\n" +
-                "    city TEXT,\n" +
-                "    country TEXT,\n" +
-                "    country_code TEXT,\n" +
-                "    device_spec json,\n" +
-                "    district_custom TEXT,\n" +
-                "    fcm_token TEXT,\n" +
-                "    first_access TIMESTAMP,\n" +
-                "    last_access TIMESTAMP,\n" +
-                "    user_declared_on TIMESTAMP,\n" +
-                "    producer_id TEXT,\n" +
-                "    state TEXT,\n" +
-                "    state_code TEXT,\n" +
-                "    state_code_custom TEXT,\n" +
-                "    state_custom TEXT,\n" +
-                "    total_launches bigint,\n" +
-                "    total_ts float,\n" +
-                "    uaspec json,\n" +
-                "    updated_date TIMESTAMP,\n" +
-                "    user_declared_district TEXT,\n" +
-                "    user_declared_state TEXT)");
-
-        stub(configMock.get("redis.host", "localhost")).toReturn("localhost");
-        stub(configMock.getInt("redis.port", 6379)).toReturn(6379);
-        stub(configMock.getInt("redis.connection.max", 2)).toReturn(2);
-        stub(configMock.getInt("redis.connection.idle.max", 2)).toReturn(2);
-        stub(configMock.getInt("redis.connection.idle.min", 1)).toReturn(1);
-        stub(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).toReturn(120);
-        stub(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).toReturn(300);
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "events_deviceprofile", new Partition(0)));
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-        stub(configMock.get("postgres.device_profile_table","device_profile")).toReturn("device_profile");
-        redisConnectMock = new RedisConnect(configMock);
-        jedisMock = redisConnectMock.getConnection(deviceStoreId);
-        deviceProfileUpdaterService = new DeviceProfileUpdaterService(configMock, redisConnectMock, postgresConnectMock);
-
-    }
-
-    @After
-    public void tearDown() {
-        redisServer.stop();
-    }
-
-    @Test
-    public void shouldupdateCache() throws Exception {
-        stub(connectionMock.createStatement()).toReturn(statementMock);
-        jedisMock.flushAll();
-
-        Type mapType = new TypeToken<Map<String, String>>(){}.getType();
-        Map<String, String> event = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType);
-        String device_id = event.get("device_id");
-
-        jedisMock.hmset(device_id, event);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        Map<String,String> cachedData = jedisMock.hgetAll(device_id);
-
-        assertEquals("232455", cachedData.get("device_id"));
-        assertEquals("Bengaluru", cachedData.get("city"));
-        assertEquals("Karnataka", cachedData.get("state"));
-        assertEquals("dev.sunbird.portal", cachedData.get("producer_id"));
-        assertEquals("IN", cachedData.get("country_code"));
-        assertEquals("Bengaluru",cachedData.get("user_declared_district"));
-        assertEquals("Karnataka",cachedData.get("user_declared_state"));
-
-        verify(deviceProfileUpdaterSinkMock, times(1)).deviceCacheUpdateSuccess();
-    }
-
-    @Test
-    public void shouldNotUpdateFordidNull() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_WITH_NO_DEVICE_ID);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        verify(deviceProfileUpdaterSinkMock, times(1)).failed();
-
-    }
-
-    @Test
-    public void shouldNotUpdateFirstAccessifPresent() throws Exception {
-        stub(connectionMock.createStatement()).toReturn(statementMock);
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-        Map<String, String> newDeviceData = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType);
-        Map<String, String> deviceDetails = new HashMap<>();
-        deviceDetails.put("firstaccess","156990957889");
-        jedisMock.hmset(newDeviceData.get("device_id"), deviceDetails);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        Map<String, String> data = jedisMock.hgetAll(newDeviceData.get("device_id"));
-        assertEquals("156990957889", data.get("firstaccess"));
-    }
-
-    @Test
-    public void shouldUpdateFirstAccessifNotPresent() throws Exception{
-        stub(connectionMock.createStatement()).toReturn(statementMock);
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-        Map<String, String> newDeviceData = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType);
-        String deviceId = newDeviceData.get("device_id");
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        Map<String, String> data = jedisMock.hgetAll(deviceId);
-        assertEquals("1568377184000", data.get("firstaccess"));
-    }
-
-    @Test
-    public void shouldAddDeviceDataToPostgres() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        ResultSet rs = statement.executeQuery("SELECT * from device_profile where device_id='232455'");
-        while(rs.next()) {
-            assertEquals("Bengaluru", rs.getString("city"));
-            assertEquals("Karnataka", rs.getString("state"));
-            assertEquals("India", rs.getString("country"));
-            assertEquals("IN", rs.getString("country_code"));
-            assertEquals("dev.sunbird.portal", rs.getString("producer_id"));
-            assertEquals("Bengaluru", rs.getString("user_declared_district"));
-            assertEquals("Karnataka", rs.getString("user_declared_state"));
-        }
-
-        verify(deviceProfileUpdaterSinkMock, times(1)).deviceDBUpdateSuccess();
-    }
-
-    @Test
-    public void shouldNotAddFirstAccessIfPresentInPostgres()throws Exception{
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-
-        String query = String.format("INSERT INTO %s (device_id, first_access) VALUES ('232455','2019-09-24 01:03:04.999');", postgres_table);
-        statement.execute(query);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        ResultSet rs=statement.executeQuery(String.format("SELECT first_access FROM %s WHERE device_id='232455';", postgres_table));
-        while(rs.next()) {
-            assertEquals("2019-09-24 01:03:04.999", rs.getString(1));
-        }
-    }
-
-    @Test
-    public void shouldAddUserDeclaredOnIfNotPresent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-
-        Type mapType = new TypeToken<Map<String, String>>(){}.getType();
-        Map<String, String> event = gson.fromJson(EventFixture.DEVICE_PROFILE_DETAILS, mapType);
-        String device_id = event.get("device_id");
-
-        jedisMock.hmset(device_id, event);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        Map<String, String> data=jedisMock.hgetAll(device_id);
-        assertEquals("1568377184000", data.get("user_declared_on"));
-    }
-
-    @Test
-    public void shouldNotAddUserDeclaredOnIfStateIsNull() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_WITH_STATE_NULL);
-
-        Type mapType = new TypeToken<Map<String, String>>(){}.getType();
-        Map<String, String> event = gson.fromJson(EventFixture.DEVICE_PROFILE_WITH_STATE_NULL, mapType);
-        String device_id = event.get("device_id");
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        Map<String, String> data=jedisMock.hgetAll(device_id);
-        assertEquals(null, data.get("user_declared_on"));
-    }
-
-    @Test
-    public void shouldNotAddUserDeclaredOnIfPresentInPostgres() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS);
-
-        String query = String.format("INSERT INTO %s (device_id, user_declared_on) VALUES ('232455','2019-09-24 01:03:04.999');", postgres_table);
-        statement.execute(query);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_on FROM %s WHERE device_id='232455';", postgres_table));
-        while(rs.next()) {
-            assertEquals("2019-09-24 01:03:04.999", rs.getString(1));
-        }
-    }
-
-    @Test
-    public void shouldInsertSpecialCharactersifPresent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS_WITH_SPECIAL_CHAR);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_state FROM %s WHERE device_id='568089542';", postgres_table));
-        while(rs.next()) {
-            assertEquals("Karnataka's", rs.getString(1));
-        }
-    }
-
-    @Test
-    public void shouldInsertSpaceCharactersifPresent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_PROFILE_DETAILS_WITH_SPACE_CHAR);
-
-        DeviceProfileUpdaterSource source = new DeviceProfileUpdaterSource(envelopeMock);
-        deviceProfileUpdaterService.process(source, deviceProfileUpdaterSinkMock);
-
-        ResultSet rs=statement.executeQuery(String.format("SELECT user_declared_district FROM %s WHERE device_id='test-did';", postgres_table));
-        while(rs.next()) {
-            assertEquals("BENGALURU URBAN SOUTH", rs.getString(1));
-        }
-    }
-}
diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
deleted file mode 100644
index adf91f919f..0000000000
--- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.ekstep.ep.samza.service.Fixtures;
-
-public class EventFixture {
-    public static final String DEVICE_PROFILE_DETAILS = "{\n" +
-            "\"fcm_token\" : \"\",\n" +
-            "\"city\" : \"Bengaluru\",\n" +
-            "\"device_id\" : \"232455\",\n" +
-            "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" +
-            "\"state\" : \"Karnataka\",\n" +
-            "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" +
-            "\"country\" : \"India\",\n" +
-            "\"country_code\" : \"IN\",\n" +
-            "\"producer_id\" : \"dev.sunbird.portal\",\n" +
-            "\"state_code_custom\" : 29,\n" +
-            "\"state_code\" : \"KA\",\n" +
-            "\"state_custom\" : \"Karnataka\",\n" +
-            "\"district_custom\" : \"Karnataka\",\n" +
-            "\"first_access\": 1568377184000,\n" +
-            "\"api_last_updated_on\": 1568377184000,\n" +
-            "\"user_declared_district\" : \"Bengaluru\",\n" +
-            "\"user_declared_state\" : \"Karnataka\"\n" +
-            "}";
-
-    public static final String DEVICE_PROFILE_WITH_STATE_NULL = "{\n" +
-            "\"fcm_token\" : \"\",\n" +
-            "\"city\" : \"Bengaluru\",\n" +
-            "\"device_id\" : \"232455\",\n" +
-            "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" +
-            "\"state\" : \"Karnataka\",\n" +
-            "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" +
-            "\"country\" : \"India\",\n" +
-            "\"country_code\" : \"IN\",\n" +
-            "\"producer_id\" : \"dev.sunbird.portal\",\n" +
-            "\"state_code_custom\" : 29,\n" +
-            "\"state_code\" : \"KA\",\n" +
-            "\"state_custom\" : \"Karnataka\",\n" +
-            "\"district_custom\" : \"Karnataka\",\n" +
-            "\"first_access\": 1568377184000,\n" +
-            "\"api_last_updated_on\": 1568377184000\n" +
-            "}";
-
-    public static final String DEVICE_PROFILE_WITH_NO_DEVICE_ID = "{\n" +
-            "\"fcm_token\" : \"\",\n" +
-            "\"city\" : \"Bengaluru\",\n" +
-            "\"device_id\" : \"\",\n" +
-            "\"device_spec\" : \"{'os':'Android 6.0','cpu':'abi: armeabi-v7a ARMv7 Processor rev 4 (v7l)','make':'Motorola XT1706'}\",\n" +
-            "\"state\" : \"Karnataka\",\n" +
-            "\"uaspec\" : \"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\n" +
-            "\"country\" : \"India\",\n" +
-            "\"country_code\" : \"IN\",\n" +
-            "\"producer_id\" : \"dev.sunbird.portal\",\n" +
-            "\"state_code_custom\" : 29,\n" +
-            "\"state_code\" : \"KA\",\n" +
-            "\"state_custom\" : \"Karnataka\",\n" +
-            "\"district_custom\" : \"Karnataka\",\n" +
-            "\"first_access\": 1568377184000,\n" +
-            "\"api_last_updated_on\": 1568377184000,\n" +
-            "\"user_declared_district\" : \"Bengaluru\",\n" +
-            "\"user_declared_state\" : \"Karnataka\"\n" +
-            "}";
-
-    public static final String DEVICE_PROFILE_DETAILS_WITH_SPECIAL_CHAR = "{\n" +
-            "\"fcm_token\" : \"\",\n" +
-            "\"device_id\" : \"568089542\",\n" +
-            "\"state\" : \"Karnataka\",\n" +
-            "\"first_access\": 1568377184000,\n" +
-            "\"api_last_updated_on\": 1568377184000,\n" +
-            "\"user_declared_district\" : \"Bengaluru\",\n" +
-            "\"user_declared_state\" : \"Karnataka's\"\n" +
-            "}";
-
-    public static final String DEVICE_PROFILE_DETAILS_WITH_SPACE_CHAR = "{\n" +
-            "\"fcm_token\" : \"\",\n" +
-            "\"device_id\" : \"test-did\",\n" +
-            "\"state\" : \"Karnataka\",\n" +
-            "\"first_access\": 1568377184000,\n" +
-            "\"api_last_updated_on\": 1568377184000,\n" +
-            "\"user_declared_district\" : \"BENGALURU URBAN SOUTH\",\n" +
-            "\"user_declared_state\" : \"Karnataka\"\n" +
-            "}";
-
-    public static final String MALFORMED_DEVICE_PROFILE_OBJECT= "{\"fcm_token\":\"\",\"city\":\"Bengaluru\",\"device_id\":\"232455\"\"device_spec”:”{‘make’}”,”state\":\"Karnataka\",\"uaspec\":\"{'agent':'Chrome','ver':'76.0.3809.132','system':'Mac OSX','raw':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}\",\"country\":\"India\",\"updated_date\":1567765886277,\"country_code\":\"IN\",\"producer_id\":\"dev.sunbird.portal\",\"state_code_custom\":\"29\",\"state_code\":\"KA\",\"state_custom\":\"Karnataka\",\"district_custom\":\"null\"}";
-}
-
diff --git a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java b/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java
deleted file mode 100644
index 58dff6322e..0000000000
--- a/data-pipeline/device-profile-updater/src/test/java/org/ekstep/ep/samza/task/DeviceProfileConfigTest.java
+++ /dev/null
@@ -1,22 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.junit.Test;
-
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.stub;
-
-public class DeviceProfileConfigTest {
-    private Config configMock;
-
-    @Test
-    public void shouldGetTheConfigs() {
-        configMock = mock(Config.class);
-        stub(configMock.get("output.malformed.topic.name", "telemetry.malformed")).toReturn("telemetry.malformed");
-        DeviceProfileUpdaterConfig config = new DeviceProfileUpdaterConfig(configMock);
-        assertTrue(config.jobName() == "DeviceProfileUpdater");
-        assertTrue(config.malformedTopic() == "telemetry.malformed");
-    }
-
-}
diff --git a/data-pipeline/distribution/pom.xml b/data-pipeline/distribution/pom.xml
deleted file mode 100644
index 455f7566ff..0000000000
--- a/data-pipeline/distribution/pom.xml
+++ /dev/null
@@ -1,176 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-    <properties>
-        <jacoco.plugin.version>0.8.5</jacoco.plugin.version>
-    </properties>
-    <artifactId>distribution</artifactId>
-
-    <packaging>pom</packaging>
-
-    <name>Distribution</name>
-
-    <!-- NOTE: These dependency declarations are only required to sort this project to the
-         end of the line in the multimodule build.
-
-         Since we only include the child1 module in our assembly, we only need to ensure this
-         distribution project builds AFTER that one...
-    -->
-    <dependencies>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>telemetry-extractor</artifactId>
-            <version>0.1.5</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>telemetry-validator</artifactId>
-            <version>0.2.2</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>de-duplication</artifactId>
-            <version>0.5.1</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>derived-de-duplication</artifactId>
-            <version>0.0.3</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>telemetry-router</artifactId>
-            <version>0.1.6</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>telemetry-redacter</artifactId>
-            <version>0.0.1</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>telemetry-location-updater</artifactId>
-            <version>0.2.3</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>de-normalization</artifactId>
-            <version>0.1.7</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>druid-events-validator</artifactId>
-            <version>0.1.4</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>events-router</artifactId>
-            <version>0.1.7</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>user-cache-updater</artifactId>
-            <version>0.0.2</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>content-cache-updater</artifactId>
-            <version>0.0.4</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>device-profile-updater</artifactId>
-            <version>0.1.8</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>assessment-aggregator</artifactId>
-            <version>0.0.4</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem.jobs</groupId>
-            <artifactId>share-events-flattener</artifactId>
-            <version>0.0.1</version>
-            <type>tar.gz</type>
-            <classifier>distribution</classifier>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>distro-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                        <configuration>
-                            <descriptors>
-                                <descriptor>src/main/assembly/src.xml</descriptor>
-                            </descriptors>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>${jacoco.plugin.version}</version>
-                <executions>
-                    <execution>
-                        <id>report-aggregate</id>
-                        <phase>prepare-package</phase>
-                        <goals>
-                            <goal>report-aggregate</goal>
-                        </goals>
-                        <configuration>
-                            <dataFileIncludes>
-                                <dataFileInclude>**/jacoco-unit.exec</dataFileInclude>
-                            </dataFileIncludes>
-                            <outputDirectory>${project.reporting.outputDirectory}/jacoco-aggregate</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-</project>
diff --git a/data-pipeline/distribution/src/main/assembly/src.xml b/data-pipeline/distribution/src/main/assembly/src.xml
deleted file mode 100644
index ce6067813c..0000000000
--- a/data-pipeline/distribution/src/main/assembly/src.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-<assembly>
-  <id>distribution</id>
-  <includeBaseDirectory>false</includeBaseDirectory>
-
-  <formats>
-    <format>tar.gz</format>
-  </formats>
-
-  <dependencySets>
-    <dependencySet>
-      <useTransitiveDependencies>false</useTransitiveDependencies>
-      <unpack>false</unpack>
-    </dependencySet>
-  </dependencySets>
-
-</assembly>
diff --git a/data-pipeline/druid-events-validator/pom.xml b/data-pipeline/druid-events-validator/pom.xml
deleted file mode 100644
index b626ef7a6b..0000000000
--- a/data-pipeline/druid-events-validator/pom.xml
+++ /dev/null
@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>druid-events-validator</artifactId>
-    <version>0.1.4</version>
-    <packaging>jar</packaging>
-    <name>DruidEventsValidator</name>
-    <description>
-        Validating DeNormalised Events Against Schema
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/druid-events-validator/src/main/assembly/src.xml b/data-pipeline/druid-events-validator/src/main/assembly/src.xml
deleted file mode 100644
index c52cb70b84..0000000000
--- a/data-pipeline/druid-events-validator/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/druid-events-validator.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:druid-events-validator</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties b/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties
deleted file mode 100644
index ecdc9fcb9c..0000000000
--- a/data-pipeline/druid-events-validator/src/main/config/druid-events-validator.properties
+++ /dev/null
@@ -1,76 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.DruidEventsValidator
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__druid_events_validator_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.DruidEventsValidatorTask
-task.inputs=kafka.__env__.telemetry.denorm
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-# Always start consuming from "kafka.__env__.telemetry.denorm"  to only process messages that are published after the job is started
-systems.kafka.samza.offset.default=upcoming
-#systems.kafka.samza.reset.offset=true
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.success.topic.name=__env__.telemetry.denorm.valid
-output.failed.topic.name=__env__.telemetry.failed
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-telemetry.schema.path=schemas/telemetry
-summary.schema.path=schemas/summary
-event.schema.file=envelope.json
-search.schema.file=search.json
-log.schema.file=log.json
-pipeline.metrics.list=success-message-count,failed-message-count,error-message-count
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 723d10d5e8..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.task.DruidEventsValidatorConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public void markSuccess() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dv_processed", true);
-        telemetry.add("type", "events");
-    }
-
-    public void markFailure(String error, DruidEventsValidatorConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dv_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        if (null != error) {
-            telemetry.add("metadata.dv_error", error);
-            telemetry.add("metadata.src", config.jobName());
-        }
-
-    }
-
-    public boolean isSummaryEvent() {
-        return eid() != null && eid().equals("ME_WORKFLOW_SUMMARY");
-    }
-
-    public boolean isSearchEvent() {
-        return "SEARCH".equalsIgnoreCase(eid());
-    }
-
-    public boolean isLogEvent() {
-        return "LOG".equalsIgnoreCase(eid());
-    }
-}
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java
deleted file mode 100644
index c711a58770..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/service/DruidEventsValidatorService.java
+++ /dev/null
@@ -1,62 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.github.fge.jsonschema.core.report.ProcessingReport;
-import com.github.fge.jsonschema.main.JsonSchemaFactory;
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.DruidEventsValidatorConfig;
-import org.ekstep.ep.samza.task.DruidEventsValidatorSink;
-import org.ekstep.ep.samza.task.DruidEventsValidatorSource;
-import org.ekstep.ep.samza.util.SchemaValidator;
-
-import static java.text.MessageFormat.format;
-
-public class DruidEventsValidatorService {
-    private static Logger LOGGER = new Logger(DruidEventsValidatorService.class);
-    private final DruidEventsValidatorConfig config;
-    private final SchemaValidator schemaValidator;
-
-    public DruidEventsValidatorService(DruidEventsValidatorConfig config, SchemaValidator schemaValidator) {
-        this.config = config;
-        this.schemaValidator = schemaValidator;
-    }
-
-    public void process(DruidEventsValidatorSource source, DruidEventsValidatorSink sink, JsonSchemaFactory jsonSchemaFactory) {
-        Event event = null;
-        try {
-            event = source.getEvent();
-            ProcessingReport report = schemaValidator.validate(event);
-            if (report.isSuccess()) {
-                event.markSuccess();
-                sink.toSuccessTopic(event);
-            } else {
-                String fieldName = this.getInvalidFieldName(report.toString());
-                LOGGER.error(null, "VALIDATION FAILED: " + report.toString());
-                sink.toFailedTopic(event, "Invalid field:" + fieldName);
-            }
-        } catch (JsonSyntaxException e) {
-            LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-            sink.toMalformedEventsTopic(source.getMessage());
-        } catch (Exception e) {
-            LOGGER.error(null, format(
-                    "EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:",
-                    event), e);
-            sink.toErrorTopic(event, e.getMessage());
-        }
-    }
-
-    private String getInvalidFieldName(String errorInfo) {
-        String[] message = errorInfo.split("reports:");
-        String noFieldNameMsg = "unable to get the field name";
-        if (message.length > 1) {
-            String[] fields = message[1].split(",");
-            if (fields.length > 2) {
-                String[] pointer = fields[3].split("\"pointer\":");
-                return pointer[1].substring(0, pointer[1].length() - 1);
-            }
-        }
-        return noFieldNameMsg;
-    }
-
-}
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java
deleted file mode 100644
index 215714a270..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorConfig.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-public class DruidEventsValidatorConfig {
-
-    private final String JOB_NAME = "DruidEventsValidator";
-    private String telemetrySchemaPath;
-    private String summarySchemaPath;
-    private String defaultSchemafile;
-    private String successTopic;
-    private String failedTopic;
-    private String malformedTopic;
-    private String searchSchemafile;
-    private String logSchemafile;
-
-    public DruidEventsValidatorConfig(Config config) {
-        successTopic = config.get("output.success.topic.name", "telemetry.denorm.valid");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        telemetrySchemaPath = config.get("telemetry.schema.path", "schemas/telemetry");
-        summarySchemaPath = config.get("summary.schema.path", "schemas/summary");
-        defaultSchemafile = config.get("event.schema.file","envelope.json");
-        searchSchemafile = config.get("search.schema.file","search.json");
-        logSchemafile = config.get("log.schema.file","log.json");
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String telemetrySchemaPath() { return telemetrySchemaPath; }
-
-    public String summarySchemaPath() { return summarySchemaPath; }
-
-    public String defaultSchemafile(){return defaultSchemafile;}
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public String searchSchemafile() { return searchSchemafile; }
-
-    public String logSchemafile() { return logSchemafile; }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java
deleted file mode 100644
index cb056b760d..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSink.java
+++ /dev/null
@@ -1,44 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-// import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class DruidEventsValidatorSink extends BaseSink {
-
-    // private JobMetrics metrics;
-    private DruidEventsValidatorConfig config;
-
-    public DruidEventsValidatorSink(MessageCollector collector, JobMetrics metrics,
-                                    DruidEventsValidatorConfig config) {
-
-        super(collector, metrics);
-        // this.metrics = metrics;
-        this.config = config;
-    }
-
-    public void toSuccessTopic(Event event) {
-        toTopic(config.successTopic(), event.mid(), event.getJson());
-        metrics.incSuccessCounter();
-    }
-
-    public void toFailedTopic(Event event, String failedMessage) {
-        event.markFailure(failedMessage, config);
-        toTopic(config.failedTopic(), event.mid(), event.getJson());
-        metrics.incFailedCounter();
-    }
-
-    public void toErrorTopic(Event event, String errorMessage) {
-        event.markFailure(errorMessage, config);
-        toTopic(config.failedTopic(), event.mid(), event.getJson());
-        metrics.incErrorCounter();
-    }
-
-    public void toMalformedEventsTopic(String message) {
-        toTopic(config.malformedTopic(), null, message);
-        metrics.incFailedCounter();
-    }
-
-}
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java
deleted file mode 100644
index 3942e8cc89..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class DruidEventsValidatorSource {
-    static Logger LOGGER = new Logger(DruidEventsValidatorSource.class);
-
-    private IncomingMessageEnvelope envelope;
-
-    public DruidEventsValidatorSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-
-}
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java
deleted file mode 100644
index 3f243bfed5..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/task/DruidEventsValidatorTask.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.service.DruidEventsValidatorService;
-import org.ekstep.ep.samza.util.SchemaValidator;
-
-import com.github.fge.jsonschema.main.JsonSchemaFactory;
-
-public class DruidEventsValidatorTask extends BaseSamzaTask {
-
-    static Logger LOGGER = new Logger(DruidEventsValidatorTask.class);
-    private DruidEventsValidatorConfig config;
-    private JobMetrics metrics;
-    private DruidEventsValidatorService service;
-    private JsonSchemaFactory jsonSchemaFactory;
-    private SchemaValidator schemaValidator;
-
-    public DruidEventsValidatorTask(Config config, TaskContext context) throws Exception {
-        init(config, context);
-    }
-
-    public DruidEventsValidatorTask() {
-
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) throws Exception {
-        this.config = new DruidEventsValidatorConfig(config);
-        metrics = new JobMetrics(context, this.config.jobName());
-        this.schemaValidator = schemaValidator == null ? new SchemaValidator(this.config) : schemaValidator;
-        service = new DruidEventsValidatorService(this.config, this.schemaValidator);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-        DruidEventsValidatorSource source = new DruidEventsValidatorSource(envelope);
-        DruidEventsValidatorSink sink = new DruidEventsValidatorSink(collector, metrics, config);
-        jsonSchemaFactory = JsonSchemaFactory.byDefault();
-        service.process(source, sink, jsonSchemaFactory);
-    }
-    
-}
diff --git a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java b/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java
deleted file mode 100644
index 7d22e8d045..0000000000
--- a/data-pipeline/druid-events-validator/src/main/java/org/ekstep/ep/samza/util/SchemaValidator.java
+++ /dev/null
@@ -1,66 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.github.fge.jackson.JsonLoader;
-import com.github.fge.jsonschema.core.exceptions.ProcessingException;
-import com.github.fge.jsonschema.core.report.ProcessingReport;
-import com.github.fge.jsonschema.main.JsonSchema;
-import com.github.fge.jsonschema.main.JsonSchemaFactory;
-import com.google.common.io.ByteStreams;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.DruidEventsValidatorConfig;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-
-public class SchemaValidator {
-
-    private final JsonSchema telemetryJsonSchema;
-    private final JsonSchema summaryJsonSchema;
-    private final JsonSchema searchEventJsonSchema;
-    private final JsonSchema logEventJsonSchema;
-    private JsonSchemaFactory schemaFactory;
-
-    public SchemaValidator(DruidEventsValidatorConfig config) throws IOException, ProcessingException {
-        this.schemaFactory = JsonSchemaFactory.byDefault();
-        String telemetrySchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(),
-                config.defaultSchemafile());
-        String summaryEventSchemapath = MessageFormat.format("{0}/{1}", config.summarySchemaPath(),
-                config.defaultSchemafile());
-        String searchEventSchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(),
-                config.searchSchemafile());
-        String logEventSchemaPath = MessageFormat.format("{0}/{1}", config.telemetrySchemaPath(),
-                config.logSchemafile());
-
-        String telemetrySchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader().
-                getResourceAsStream(telemetrySchemaPath)));
-        String summarySchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader().
-                getResourceAsStream(summaryEventSchemapath)));
-        String searchEventSchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader().
-                getResourceAsStream(searchEventSchemaPath)));
-        String logEventSchema = new String(ByteStreams.toByteArray(this.getClass().getClassLoader().
-                getResourceAsStream(logEventSchemaPath)));
-
-        this.telemetryJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(telemetrySchema));
-        this.summaryJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(summarySchema));
-        this.searchEventJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(searchEventSchema));
-        this.logEventJsonSchema = schemaFactory.getJsonSchema(JsonLoader.fromString(logEventSchema));
-    }
-
-    public ProcessingReport validate(Event event) throws IOException, ProcessingException {
-
-        JsonNode eventJson = JsonLoader.fromString(event.getJson());
-        ProcessingReport report;
-        if (event.isSearchEvent()) {
-            report = searchEventJsonSchema.validate(eventJson);
-        } else if (event.isSummaryEvent()) {
-            report = summaryJsonSchema.validate(eventJson);
-        } else if (event.isLogEvent()) {
-            report = logEventJsonSchema.validate(eventJson);
-        } else {
-            report = telemetryJsonSchema.validate(eventJson);
-        }
-        return report;
-    }
-
-}
diff --git a/data-pipeline/druid-events-validator/src/main/resources/log4j.xml b/data-pipeline/druid-events-validator/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json
deleted file mode 100644
index d855ee7b51..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.sunbird.org/summary/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.sunbird.org/summary/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json
deleted file mode 100644
index f069aa65b6..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/collectiondata.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "name": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/name",
-      "type": "string"
-    },
-    "objecttype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/objecttype",
-      "type": "string"
-    },
-    "contenttype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/contenttype",
-      "type": "string"
-    },
-    "mediatype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/mediatype",
-      "type": "string"
-    },
-    "language": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/language",
-      "type": "array"
-    },
-    "medium": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/medium",
-      "type": "array"
-    },
-    "gradelevel": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/gradelevel",
-      "type": "array"
-    },
-    "subject": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/subject",
-      "type": "array"
-    },
-    "mimetype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/mimetype",
-      "type": "string"
-    },
-    "createdby": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/createdby",
-      "type": "string"
-    },
-    "createdfor": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/createdfor",
-      "type": "array"
-    },
-    "framework": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/framework",
-      "type": "string"
-    },
-    "board": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/board",
-      "type": "string"
-    },
-    "status": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/status",
-      "type": "string"
-    },
-    "pkgversion": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/pkgversion",
-      "type": "number"
-    },
-    "lastsubmittedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastsubmittedon",
-      "type": "number"
-    },
-    "lastpublishedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastpublishedon",
-      "type": "number"
-    },
-    "lastupdatedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastupdatedon",
-      "type": "number"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json
deleted file mode 100644
index 6b2ee98324..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/common.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.sunbird.org/summary/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/summary/ver",
-      "type": "string"
-    },
-    "mid": {
-      "id": "http://api.sunbird.org/summary/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "context": {
-      "$ref": "resource:/schemas/summary/context.json"
-    },
-    "object": {
-      "$ref": "resource:/schemas/summary/object.json"
-    },
-    "dimensions": {
-      "$ref": "resource:/schemas/summary/dimension.json"
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    },
-    "devicedata": {
-      "$ref": "resource:/schemas/summary/devicedata.json"
-    },
-    "userdata": {
-      "$ref": "resource:/schemas/summary/userdata.json"
-    },
-    "contentdata": {
-      "$ref": "resource:/schemas/summary/contentdata.json"
-    },
-    "collectiondata": {
-      "$ref": "resource:/schemas/summary/collectiondata.json"
-    },
-    "derivedlocationdata": {
-      "$ref": "resource:/schemas/summary/derivedlocationdata.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json
deleted file mode 100644
index e1fa75cc75..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/contentdata.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "name": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/name",
-      "type": "string"
-    },
-    "objecttype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/objecttype",
-      "type": "string"
-    },
-    "contenttype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/contenttype",
-      "type": "string"
-    },
-    "mediatype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/mediatype",
-      "type": "string"
-    },
-    "language": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/language",
-      "type": "array"
-    },
-    "medium": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/medium",
-      "type": "array"
-    },
-    "gradelevel": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/gradelevel",
-      "type": "array"
-    },
-    "createdby": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/createdby",
-      "type": "string"
-    },
-    "createdfor": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/createdfor",
-      "type": "array"
-    },
-    "subject": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/subject",
-      "type": "array"
-    },
-    "mimetype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/mimetype",
-      "type": "string"
-    },
-    "framework": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/framework",
-      "type": "string"
-    },
-    "board": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/board",
-      "type": "string"
-    },
-    "status": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/status",
-      "type": "string"
-    },
-    "pkgversion": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/pkgversion",
-      "type": "number"
-    },
-    "lastsubmittedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastsubmittedon",
-      "type": "number"
-    },
-    "lastpublishedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastpublishedon",
-      "type": "number"
-    },
-    "lastupdatedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastupdatedon",
-      "type": "number"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json
deleted file mode 100644
index 4d039d4abc..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/context.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "pdata": {
-      "$ref": "resource:/schemas/summary/pdata.json"
-    },
-    "granularity": {
-      "id": "http://api.sunbird.org/summary/context/granularity",
-      "type": "string"
-    },
-    "date_range": {
-      "$ref": "resource:/schemas/summary/datarange.json"
-    },
-    "rollup": {
-      "$ref": "resource:/schemas/summary/rollup.json"
-    },
-    "did": {
-      "id": "http://api.sunbird.org/summary/context/did",
-      "type": "string"
-    },
-    "cdata": {
-      "$ref": "resource:/schemas/summary/cdata.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json
deleted file mode 100644
index 036c7cc798..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/datarange.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "from": {
-      "id": "http://api.sunbird.org/summary/datarange/from",
-      "type": "number"
-    },
-    "to": {
-      "id": "http://api.sunbird.org/summary/datarange/to",
-      "type": "number"
-    } 
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json
deleted file mode 100644
index e121bd4f46..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/derivedlocationdata.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/state",
-      "type": "string"
-    },
-    "district":{
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/district",
-      "type": "string"
-    },
-    "from":{
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/from",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json
deleted file mode 100644
index 555aa35265..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/devicedata.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "uaspec": {
-      "$ref": "resource:/schemas/summary/uaspec.json"
-    },
-    "firstaccess": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/firstaccess",
-      "type": "number"
-    },
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/state",
-      "type": "string"
-    },
-    "statecode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecode",
-      "type": "string"
-    },
-    "city": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/city",
-      "type": "string"
-    },
-    "countrycode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/countrycode",
-      "type": "string"
-    },
-    "country": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/country",
-      "type": "string"
-    },
-    "os": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/os",
-      "type": "string"
-    },
-    "make": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/make",
-      "type": "string"
-    },
-    "id": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/id",
-      "type": "string"
-    },
-    "mem": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/mem",
-      "type": "string"
-    },
-    "idisk": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/idisk",
-      "type": "string"
-    },
-    "edisk": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/edisk",
-      "type": "string"
-    },
-    "scrn": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/scrn",
-      "type": "string"
-    },
-    "camera": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/camera",
-      "type": "string"
-    },
-    "cpu": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/cpu",
-      "type": "string"
-    },
-    "sims": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/sims",
-      "type": "string"
-    },
-    "iso3166statecode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/iso3166statecode",
-      "type": "string"
-    },
-    "statecustomcode":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecustomcode",
-      "type": "string"
-    },
-    "statecustomname":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecustomname",
-      "type": "string"
-    },
-    "districtcustom":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/districtcustom",
-      "type": "string"
-    },
-    "userdeclared": {
-      "type": "object",
-      "properties": {
-        "state": {
-          "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/state",
-          "type": "string"
-        },
-        "district": {
-          "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/district",
-          "type": "string"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json
deleted file mode 100644
index 65a5624207..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "channel": {
-      "id": "http://api.sunbird.org/summary/dimension/channel",
-      "type":"string"
-    },
-    "sid": {
-      "id": "http://api.sunbird.org/summary/dimension/sid",
-      "type": "string"
-    },
-    "did": {
-      "id": "http://api.sunbird.org/summary/dimension/did",
-      "type": "string"
-    },
-    "pdata": {
-      "$ref": "resource:/schemas/summary/dimension_pdata.json"
-    },
-    "type": {
-      "id": "http://api.sunbird.org/telemetry/dimension/type",
-      "type": "string"
-    },
-    "mode": {
-      "id": "http://api.sunbird.org/telemetry/dimension/mode",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json
deleted file mode 100644
index 2467bd693d..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/dimension_pdata.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.sunbird.org/summary/pdata/id",
-      "type": "string"
-    },
-    "pid": {
-      "id": "http://api.sunbird.org/summary/pdata/pid",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/summary/pdata/ver",
-      "type": "string"
-    }
-  },
-  "required": [
-    "id"
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json
deleted file mode 100644
index b22367f837..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/envelope.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "id":"http://api.sunbird.org/summary/envelope",
-  "type":"object",
-  "required":[
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "context",
-    "edata"
-  ],
-  "allOf":[
-    {
-      "$ref":"resource:/schemas/summary/common.json"
-    },
-    {
-      "properties":{
-        "eid":{
-          "id":"http://api.sunbird.org/telemetry/mid",
-          "type":"string",
-          "minLength": 1
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json
deleted file mode 100644
index d8a85ee0ae..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/me_workflow_summary.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-  "id": "http://api.sunbird.org/summary/me_workflow_summary",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "syncts",
-    "mid",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/schemas/summary/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.sunbird.org/summary/eid",
-          "enum": [
-            "ME_WORKFLOW_SUMMARY"
-          ]
-        },
-        "edata": {
-          "id": "http://api.sunbird.org/summary/edata",
-          "type": "object"
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json
deleted file mode 100644
index bfebbd3c6c..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/object.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.sunbird.org/summary/object/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.sunbird.org/summary/object/type",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/summary/object/ver",
-      "type": "string"
-    },
-    "rollup": {
-      "$ref": "resource:/schemas/summary/rollup.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json
deleted file mode 100644
index dd29b0cc45..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/pdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.sunbird.org/summary/pdata/id",
-      "type": "string"
-    },
-    "model": {
-      "id": "http://api.sunbird.org/summary/pdata/model",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/summary/pdata/ver",
-      "type": "string"
-    }
-  },
-  "required": ["id"]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json
deleted file mode 100644
index baf81742a3..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/rollup.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "l1": {
-      "id": "http://api.sunbird.org/summary/context/rollup/l1",
-      "type": "string"
-    },
-    "l2": {
-      "id": "http://api.sunbird.org/summary/context/rollup/l2",
-      "type": "string"
-    },
-    "l3": {
-      "id": "http://api.sunbird.org/summary/context/rollup/l3",
-      "type": "string"
-    },
-    "l4": {
-      "id": "http://api.sunbird.org/summary/context/rollup/l4",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json
deleted file mode 100644
index 138155b0b3..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/uaspec.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "agent": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/agent",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/ver",
-      "type": "string"
-    },
-    "system": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/system",
-      "type": "string"
-    },
-    "platform": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/platform",
-      "type": "string"
-    },
-    "raw": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/raw",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json
deleted file mode 100644
index 8e96683f62..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/summary/userdata.json
+++ /dev/null
@@ -1,29 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "gradelist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/gradelist",
-      "type": "array"
-    },
-    "languagelist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/languagelist",
-      "type": "array"
-    },
-    "subjectlist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/subjectlist",
-      "type": "array"
-    },
-    "usertype": {
-      "id": "http://api.sunbird.org/telemetry/userdata/usertype",
-      "type": "string"
-    },
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/userdata/state",
-      "type": "string"
-    },
-    "district": {
-      "id": "http://api.sunbird.org/telemetry/userdata/district",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json
deleted file mode 100644
index f069aa65b6..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/collectiondata.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "name": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/name",
-      "type": "string"
-    },
-    "objecttype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/objecttype",
-      "type": "string"
-    },
-    "contenttype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/contenttype",
-      "type": "string"
-    },
-    "mediatype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/mediatype",
-      "type": "string"
-    },
-    "language": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/language",
-      "type": "array"
-    },
-    "medium": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/medium",
-      "type": "array"
-    },
-    "gradelevel": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/gradelevel",
-      "type": "array"
-    },
-    "subject": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/subject",
-      "type": "array"
-    },
-    "mimetype": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/mimetype",
-      "type": "string"
-    },
-    "createdby": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/createdby",
-      "type": "string"
-    },
-    "createdfor": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/createdfor",
-      "type": "array"
-    },
-    "framework": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/framework",
-      "type": "string"
-    },
-    "board": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/board",
-      "type": "string"
-    },
-    "status": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/status",
-      "type": "string"
-    },
-    "pkgversion": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/pkgversion",
-      "type": "number"
-    },
-    "lastsubmittedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastsubmittedon",
-      "type": "number"
-    },
-    "lastpublishedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastpublishedon",
-      "type": "number"
-    },
-    "lastupdatedon": {
-      "id": "http://api.sunbird.org/telemetry/collectiondata/lastupdatedon",
-      "type": "number"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json
deleted file mode 100644
index a54d9cd888..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/common.json
+++ /dev/null
@@ -1,28 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ver": {
-      "id": "http://api.sunbird.org/telemetry/ver",
-      "type": "string"
-
-    },
-    "devicedata": {
-      "$ref": "resource:/schemas/telemetry/devicedata.json"
-    },
-    "contentdata": {
-      "$ref": "resource:/schemas/telemetry/contentdata.json"
-    },
-    "collectiondata": {
-      "$ref": "resource:/schemas/telemetry/collectiondata.json"
-    },
-    "dialcodedata": {
-      "$ref": "resource:/schemas/telemetry/dialcodedata.json"
-    },
-    "userdata": {
-      "$ref": "resource:/schemas/telemetry/userdata.json"
-    },
-    "derivedlocationdata": {
-      "$ref": "resource:/schemas/telemetry/derivedlocationdata.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json
deleted file mode 100644
index a21f612cab..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/contentdata.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "name": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/name",
-      "type": "string"
-    },
-    "objecttype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/objecttype",
-      "type": "string"
-    },
-    "contenttype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/contenttype",
-      "type": "string"
-    },
-    "mediatype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/mediatype",
-      "type": "string"
-    },
-    "language": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/language",
-      "type": "array"
-    },
-    "medium": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/medium",
-      "type": "array"
-    },
-    "gradelevel": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/gradelevel",
-      "type": "array"
-    },
-    "subject": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/subject",
-      "type": "array"
-    },
-    "mimetype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/mimetype",
-      "type": "string"
-    },
-    "createdby": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/createdby",
-      "type": "string"
-    },
-    "createdfor": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/createdfor",
-      "type": "array"
-    },
-    "framework": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/framework",
-      "type": "string"
-    },
-    "board": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/board",
-      "type": "string"
-    },
-    "status": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/status",
-      "type": "string"
-    },
-    "pkgversion": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/pkgversion",
-      "type": "number"
-    },
-    "lastsubmittedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastsubmittedon",
-      "type": "number"
-    },
-    "lastpublishedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastpublishedon",
-      "type": "number"
-    },
-    "lastupdatedon": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/lastupdatedon",
-      "type": "number"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json
deleted file mode 100644
index e121bd4f46..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/derivedlocationdata.json
+++ /dev/null
@@ -1,17 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/state",
-      "type": "string"
-    },
-    "district":{
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/district",
-      "type": "string"
-    },
-    "from":{
-      "id": "http://api.sunbird.org/telemetry/derivedlocationdata/from",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json
deleted file mode 100644
index e24b79e01a..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/devicedata.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "uaspec": {
-      "$ref": "resource:/schemas/telemetry/uaspec.json"
-    },
-    "firstaccess": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/firstaccess",
-      "type": "number"
-    },
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/state",
-      "type": "string"
-    },
-    "statecode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecode",
-      "type": "string"
-    },
-    "city": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/city",
-      "type": "string"
-    },
-    "countrycode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/countrycode",
-      "type": "string"
-    },
-    "country": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/country",
-      "type": "string"
-    },
-    "os": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/os",
-      "type": "string"
-    },
-    "make": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/make",
-      "type": "string"
-    },
-    "id": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/id",
-      "type": "string"
-    },
-    "mem": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/mem",
-      "type": "string"
-    },
-    "idisk": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/idisk",
-      "type": "string"
-    },
-    "edisk": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/edisk",
-      "type": "string"
-    },
-    "scrn": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/scrn",
-      "type": "string"
-    },
-    "camera": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/camera",
-      "type": "string"
-    },
-    "cpu": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/cpu",
-      "type": "string"
-    },
-    "sims": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/sims",
-      "type": "string"
-    },
-    "iso3166statecode": {
-      "id": "http://api.sunbird.org/telemetry/devicedata/iso3166statecode",
-      "type": "string"
-    },
-    "statecustomcode":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecustomcode",
-      "type": "string"
-    },
-    "statecustomname":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/statecustomname",
-      "type": "string"
-    },
-    "districtcustom":{
-      "id": "http://api.sunbird.org/telemetry/devicedata/districtcustom",
-      "type": "string"
-    },
-    "userdeclared": {
-      "type": "object",
-      "properties": {
-        "state": {
-          "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/state",
-          "type": "string"
-        },
-        "district": {
-          "id": "http://api.sunbird.org/telemetry/devicedata/userdeclared/district",
-          "type": "string"
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json
deleted file mode 100644
index 06e979a5ff..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/dialcodedata.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  "type": [
-    "object",
-    "array"
-  ],
-  "properties": {
-    "channel": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/channel",
-      "type": "string"
-    },
-    "identifier": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/identifier",
-      "type": "string"
-    },
-    "batchcode": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/batchcode",
-      "type": "string"
-    },
-    "publisher": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/publisher",
-      "type": "string"
-    },
-    "generatedon": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/generatedon",
-      "type": "number"
-    },
-    "publishedon": {
-      "id": "http://api.sunbird.org/telemetry/dialcodedata/publishedon",
-      "type": "number"
-    },
-    "objecttype": {
-      "id": "http://api.sunbird.org/telemetry/contentdata/objecttype",
-      "type": "string"
-    }
-  },
-  "items": {
-    "type": "object",
-    "properties": {
-      "channel": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/channel",
-        "type": "string"
-      },
-      "identifier": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/identifier",
-        "type": "string"
-      },
-      "batchcode": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/batchcode",
-        "type": "string"
-      },
-      "publisher": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/publisher",
-        "type": "string"
-      },
-      "generatedon": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/generatedon",
-        "type": "number"
-      },
-      "publishedon": {
-        "id": "http://api.sunbird.org/telemetry/dialcodedata/publishedon",
-        "type": "number"
-      },
-      "objecttype": {
-        "id": "http://api.sunbird.org/telemetry/contentdata/objecttype",
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json
deleted file mode 100644
index 8eef044eed..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/envelope.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "id":"http://api.sunbird.org/telemetry/envelope",
-  "type":"object",
-  "allOf":[
-    {
-      "$ref":"resource:/schemas/telemetry/common.json"
-    },
-    {
-      "properties":{
-        "eid":{
-          "id":"http://api.sunbird.org/telemetry/mid",
-          "type":"string",
-          "minLength": 1
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json
deleted file mode 100644
index c6f61d886d..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/filter.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-  },
-  "anyOf": [
-    {
-      "not": {
-        "required": [
-          "dialCodes"
-        ]
-      }
-    }
-  ],
-  "additionalProperties": true
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json
deleted file mode 100644
index 114b80780c..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/log.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
-  "id": "http://api.sunbird.org/telemetry/log",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/schemas/telemetry/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.sunbird.org/telemetry/eid",
-          "enum": [
-            "LOG"
-          ]
-        },
-        "edata": {
-          "id": "http://api.sunbird.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "level",
-            "message"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.sunbird.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "level": {
-              "id": "http://api.sunbird.org/telemetry/edata/level",
-              "type": "string"
-            },
-            "message": {
-              "id": "http://api.sunbird.org/telemetry/edata/message",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.sunbird.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "params": {
-              "id": "http://api.sunbird.org/telemetry/edata/params",
-              "type": "array",
-              "items": {
-                "type": "object"
-              },
-              "properties": {
-                  "rid": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/rid",
-                    "type": "string"
-                  },
-                  "uip": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/uip",
-                    "type": "string"
-                  },
-                  "title": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/title",
-                    "type": "string"
-                  },
-                  "category": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/category",
-                    "type": "string"
-                  },
-                  "url": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/url",
-                    "type": "string"
-                  },
-                  "size": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/size",
-                    "type": "number"
-                  },
-                  "duration": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/duration",
-                    "type": "number"
-                  },
-                  "status": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/status",
-                    "type": "number"
-                  },
-                  "protocol": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/protocol",
-                    "type": "string"
-                  },
-                  "method": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/method",
-                    "type": "string"
-                  },
-                  "req": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/req",
-                    "type": "string"
-                  },
-                  "consumer_id": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/consumer_id",
-                    "type": "string"
-                  },
-                  "ver": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/ver",
-                    "type": "string"
-                  },
-                  "events_count": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/events_count",
-                    "type": "string"
-                  },
-                  "sync_status": {
-                    "id": "http://api.sunbird.org/telemetry/edata/params/sync_status",
-                    "type": "string"
-                  }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json
deleted file mode 100644
index 73ddce9313..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/search.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "id": "http://api.sunbird.org/telemetry/search",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/schemas/telemetry/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.sunbird.org/telemetry/eid",
-          "enum": [
-            "SEARCH"
-          ]
-        },
-        "edata": {
-          "id": "http://api.sunbird.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "query",
-            "size",
-            "topn"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.sunbird.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "query": {
-              "id": "http://api.sunbird.org/telemetry/edata/query",
-              "type": "string"
-            },
-            "sort": {
-              "id": "http://api.sunbird.org/telemetry/edata/sort",
-              "type": "object"
-            },
-            "correlationid": {
-              "id": "http://api.sunbird.org/telemetry/edata/correlationid",
-              "type": "string"
-            },
-            "size": {
-              "id": "http://api.sunbird.org/telemetry/edata/size",
-              "type": "number"
-            },
-            "filters": {
-              "$ref": "resource:/schemas/telemetry/filter.json"
-            },
-            "topn": {
-              "id": "http://api.sunbird.org/telemetry/edata/topn",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json
deleted file mode 100644
index 138155b0b3..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/uaspec.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "agent": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/agent",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/ver",
-      "type": "string"
-    },
-    "system": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/system",
-      "type": "string"
-    },
-    "platform": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/platform",
-      "type": "string"
-    },
-    "raw": {
-      "id": "http://api.sunbird.org/telemetry/uaspec/raw",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json b/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json
deleted file mode 100644
index bd80c89196..0000000000
--- a/data-pipeline/druid-events-validator/src/main/resources/schemas/telemetry/userdata.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "gradelist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/gradelist",
-      "type": "array"
-    },
-    "languagelist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/languagelist",
-      "type": "array"
-    },
-    "subjectlist": {
-      "id": "http://api.sunbird.org/telemetry/userdata/subjectlist",
-      "type": "array"
-    },
-    "usertype": {
-      "id": "http://api.sunbird.org/telemetry/userdata/usertype",
-      "type": "string"
-    },
-    "state": {
-      "id": "http://api.sunbird.org/telemetry/userdata/state",
-      "type": "string"
-    },
-    "district": {
-      "id": "http://api.sunbird.org/telemetry/userdata/district",
-      "type": "string"
-    },
-    "usersignintype": {
-      "id": "http://api.sunbird.org/telemetry/usersignintype/district",
-      "type": "string"
-    },
-    "userlogintype": {
-      "id": "http://api.sunbird.org/telemetry/userdata/userlogintype",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java
deleted file mode 100644
index d584ec988a..0000000000
--- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/SummaryV1.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-public class SummaryV1 {
-    /**
-     * Valid Summary Event
-     */
-    public static final String VALID_EVENT = "{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":1551245418895,\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":{\"did\":\"518bab62deac8129f563077b4f1ba516\",\"pdata\":{\"id\":\"staging.diksha.portal\",\"ver\":\"1.11.0\",\"pid\":\"sunbird-portal.contenteditor.contentplayer\"},\"sid\":\"nlYPjt8_GoxB8LDtldOChvz7YH3odbKH\",\"channel\":\"0124784842112040965\",\"type\":\"content\",\"mode\":\"edit\"},\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":[],\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}";
-
-    /**
-     * Invalid dimension data type Event
-     */
-    public static final String INVALID_EVENT= "{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":1551245418895,\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":[],\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":[],\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}";
-
-    /**
-     * Invalid Tags data type
-     */
-    public static final String INVALID_EVENT_CASE_1 ="{\"eid\":\"ME_WORKFLOW_SUMMARY\",\"ets\":1551409245701,\"syncts\":\"1551245418895\",\"ver\":\"1.0\",\"mid\":\"4E12E343B0FD99D17490C1BD0DB69B4F\",\"uid\":\"8ec91293-21d8-4af3-aeaa-275b35dc8c98\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkflowSummarizer\"},\"granularity\":\"SESSION\",\"date_range\":{\"from\":1551244161206,\"to\":1551245418814},\"rollup\":{},\"cdata\":[{\"id\":\"9d282a8cbaf1546462a4851691b3bc00\",\"type\":\"ContentSession\"}]},\"dimensions\":{\"did\":\"518bab62deac8129f563077b4f1ba516\",\"pdata\":{\"id\":\"staging.diksha.portal\",\"ver\":\"1.11.0\",\"pid\":\"sunbird-portal.contenteditor.contentplayer\"},\"sid\":\"nlYPjt8_GoxB8LDtldOChvz7YH3odbKH\",\"channel\":\"0124784842112040965\",\"type\":\"content\",\"mode\":\"edit\"},\"edata\":{\"eks\":{\"interact_events_per_min\":6.57,\"start_time\":1551244161206,\"interact_events_count\":63,\"item_responses\":[],\"end_time\":1551245418814,\"events_summary\":[{\"id\":\"START\",\"count\":1},{\"id\":\"IMPRESSION\",\"count\":21},{\"id\":\"INTERACT\",\"count\":63},{\"id\":\"END\",\"count\":1}],\"page_summary\":[{\"id\":\"d9a95168-3e15-4aac-91e6-b8505116a497\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":193.83,\"visit_count\":1},{\"id\":\"c0e843b3-ec91-42ed-8e21-21250d1cb205\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":15.07,\"visit_count\":1},{\"id\":\"aaada2d5-6199-40d5-a85c-e2c778e8b18e\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":2.06,\"visit_count\":1},{\"id\":\"5d710ee4-b589-49bb-ad1d-8c1d08e6a83a\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":17.33,\"visit_count\":1},{\"id\":\"62481bee-3d01-4d5c-b0e9-50b837056604\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":12.75,\"visit_count\":1},{\"id\":\"6cb0248e-b9ae-4beb-817a-0ec86b638d07\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":119.6,\"visit_count\":1},{\"id\":\"36a9709b-e218-49b4-a19c-1a44947ab84d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":113.67,\"visit_count\":1},{\"id\":\"dcd4adc7-7959-45c5-8aa5-0a75b269850d\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":1365.73,\"visit_count\":1},{\"id\":\"c40dd5a6-6375-42c7-8ce1-dba65a8a1826\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":5.36,\"visit_count\":1},{\"id\":\"2b0278a0-6673-4ba0-9ba9-17b5bcc619d0\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":35.19,\"visit_count\":1},{\"id\":\"244d4eee-17d4-4bad-8be3-0a8849dea53c\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":19.36,\"visit_count\":1},{\"id\":\"5786c403-d267-4329-a61a-3dccc466a4ed\",\"type\":\"workflow\",\"env\":\"contentplayer\",\"time_spent\":9.07,\"visit_count\":1}],\"time_diff\":1257.61,\"telemetry_version\":\"3.0\",\"env_summary\":[{\"env\":\"contentplayer\",\"time_spent\":1909.02,\"count\":1}],\"time_spent\":575.19}},\"tags\":{},\"object\":{\"id\":\"do_21268948156283289611498\",\"type\":\"Content\",\"ver\":\"2\"}}";
-    /**
-     * Invalid Tags data type
-     */
-    public static final String INVALID_SUMMARY ="{\"eid\":\"ME_MONITOR_SUMMARY\",\"ets\":1553830612140,\"syncts\":1553817602545,\"ver\":\"2.1\",\"mid\":\"6ED077D95C3B097547544DFF127F647E\",\"uid\":\"\",\"channel\":\"\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"MonitorSummarizer\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1553817602545,\"to\":1553830609569}},\"dimensions\":{\"period\":20190329},\"edata\":{\"eks\":{\"total_events_generated\":2013,\"jobs_failed_count\":0,\"jobs_summary\":[{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"WorkFlowSummaryModel\",\"tag\":\"\",\"time_taken\":8,\"output_count\":52,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"LearnerSessionSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":21,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"ItemSummaryModel\",\"tag\":\"\",\"time_taken\":4,\"output_count\":6,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":21,\"day\":20190329},{\"model\":\"ItemUsageSummaryModel\",\"tag\":\"\",\"time_taken\":6,\"output_count\":24,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":6,\"day\":20190329},{\"model\":\"UsageSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":68,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":21,\"day\":20190329},{\"model\":\"UpdateItemSummaryDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":96,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":24,\"day\":20190329},{\"model\":\"WorkFlowUsageSummaryModel\",\"tag\":\"\",\"time_taken\":5,\"output_count\":348,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":52,\"day\":20190329},{\"model\":\"DeviceSummaryModel\",\"tag\":\"\",\"time_taken\":3,\"output_count\":3,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38570,\"day\":20190329},{\"model\":\"UpdateDeviceProfileDB\",\"tag\":\"\",\"time_taken\":0,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":3,\"day\":20190329},{\"model\":\"UpdateWorkFlowUsageDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":1392,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":348,\"day\":20190329},{\"model\":\"UpdateWorkFlowUsageMetricsModel\",\"tag\":\"\",\"time_taken\":0,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":348,\"day\":20190329},{\"model\":\"UpdatePortalMetrics\",\"tag\":\"\",\"time_taken\":1,\"output_count\":1,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"DialcodeUsageSummaryModel\",\"tag\":\"\",\"time_taken\":12,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":38518,\"day\":20190329},{\"model\":\"ETBCoverageSummaryModel\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"UpdateDialcodeUsageDB\",\"tag\":\"\",\"time_taken\":1,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"VideoStreamingJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329},{\"model\":\"DataExhaustJob\",\"tag\":\"\",\"time_taken\":0,\"output_count\":0,\"model_id\":\"AnalyticsDataPipeline\",\"status\":\"SUCCESS\",\"input_count\":0,\"day\":20190329}],\"total_ts\":52,\"jobs_completed_count\":38,\"jobs_start_count\":38}},\"ts\":\"2019-03-29T03:36:52.140+0000\",\"ldata\":{\"country_code\":\"\",\"country\":\"\",\"city\":\"\",\"state\":\"\",\"state_code\":\"\"},\"flags\":{\"ldata_retrieved\":false}}";
-
-    public static final String UASPEC_SUMMARY="{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{\"platform\":123,\"ver\":\"\"},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-}
diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java
deleted file mode 100644
index 82f2909332..0000000000
--- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/fixtures/TelemetryV3.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-public class TelemetryV3 {
-    /**
-     * Valid event with all proper data type
-     */
-    public static final String VALID_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-    /**
-     * Passing Invalid event having wrong `pkgVersion` type
-     */
-    public static final String INVALID_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":\"1\",\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-    /**
-     * Passing Invalid event having wrong `gradelist` data type
-     */
-    public static final String INVALID_USERDATA = "{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},\"userdata\":{\"gradelist\":{},\"languagelist\":[],\"subjectlist\":[],\"usertype\":\"\",\"state\":\"\",\"district\":\"\"}}";
-
-    /**
-     * Passing Invalid Device data having wrong `firstAccess` data type
-     */
-    public static final String INVALID_DEVICEDATA_CASE_1 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":\"1545920698694\"},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-    /**
-     * Passing Invalid Device data having wrong `uaspec` data type
-     */
-    public static final String INVALID_DEVICEDATA_CASE_2 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":\"India\",\"city\":\"Banglore\",\"countrycode\":\"IND\",\"state\":\"Karnataka\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":[],\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-    /**
-     * Passing Invalid Device data having wrong `country` data type and missing few fields
-     */
-    public static final String INVALID_DEVICEDATA_CASE_3 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"KA\",\"country\":958743,\"city\":\"Banglore\",\"countrycode\":\"IND\",\"statecode\":\"\",\"districtcustom\":\"\",\"uaspec\":[],\"firstaccess\":1545920698694},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-    /**
-     * Passing Invalid Device data having wrong `firstAccess` data type to float format
-     */
-    public static final String INVALID_DEVICEDATA_CASE_4 = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":12},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":[\"Assamese\"],\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-
-    /**
-     * Passing Invalid Event Having wrong data type format in Languages field
-     */
-    public static final String INVALID_CONTENTDATA = "{\"actor\":{\"type\":\"User\",\"id\":\"4c4530df-0d4f-42a5-bd91-0366716c8c24\"},\"edata\":{\"id\":\"content-detail\",\"type\":\"OTHER\",\"pageid\":\"content-detail\",\"subtype\":\"detail\",\"extra\":{\"values\":[{\"isDownloaded\":true,\"isUpdateAvailable\":false}]}},\"eid\":\"INTERACT\",\"ver\":\"3.0\",\"ets\":1551344686294,\"context\":{\"pdata\":{\"ver\":\"2.0.localstaging-debug\",\"pid\":\"sunbird.app\",\"id\":\"staging.diksha.app\"},\"channel\":\"01231711180382208027\",\"env\":\"home\",\"did\":\"6c61348dc9841f27c96f4887b64ee1f777d74c38\",\"sid\":\"cef2d0be-83fc-4988-8ad9-1b72399e6d3a\",\"cdata\":[]},\"mid\":\"3318a611-50fa-4ae9-9167-7b4390a62b9f\",\"object\":{\"id\":\"do_21228031946955980819\",\"type\":\"Worksheet\",\"version\":\"1.0\",\"rollup\":{\"l4\":\"do_21270636501657190413450\",\"l1\":\"do_21270636097196032013440\",\"l2\":\"do_21270636501655552013444\",\"l3\":\"do_21270636501657190413448\"}},\"tags\":[],\"syncts\":1551344699388,\"@timestamp\":\"2019-02-28T09:04:59.388Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true,\"user_location_retrieved\":false,\"content_data_retrieved\":true,\"user_data_retrieved\":true,\"device_data_retrieved\":true},\"type\":\"events\",\"ts\":\"2019-02-28T09:04:46.294+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\",\"uaspec\":{},\"firstaccess\":125465324536475},\"userdata\":{\"district\":\"\",\"state\":\"\",\"subject\":[\"English\"],\"grade\":[\"KG\",\"Class 1\",\"Class 2\",\"Class 3\",\"Class 4\",\"Class 5\",\"Class 6\",\"Class 7\",\"Class 8\",\"Class 9\",\"Class 10\",\"Class 11\",\"Class 12\",\"Other\"],\"language\":[\"English\",\"Gujarati\",\"Hindi\"]},\"contentdata\":{\"pkgversion\":1,\"language\":{},\"lastpublishedon\":1499851249497,\"contenttype\":\"Resource\",\"lastupdatedon\":1499851152176,\"framework\":\"NCF\",\"name\":\"Test review process\",\"mimetype\":\"application/vnd.ekstep.ecml-archive\",\"objecttype\":\"Content\",\"mediatype\":\"content\",\"status\":\"Live\"}}";
-
-    /**
-     *
-     */
-    public static final String INVALID_DIALCODEDATA ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"objecttype\":1}}";
-
-    /**
-     * case sensitive dialcode key validation
-     *
-     */
-    public static final String INVALID_DIALCODE_KEY ="{\"eid\":\"SEARCH\",\"ets\":1551977394372,\"ver\":\"3.0\",\"mid\":\"LP.1551977394372.d18d0bdf-9fbe-42ff-ac2c-3cb7306462a6\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\",\"did\":\"530f35ec69f30432bd5e87eee4e7d9a82da0e129\"},\"edata\":{\"size\":14,\"query\":\"\",\"filters\":{\"channels\":\"yes\",\"dialCodes\":\"\",\"objectType\":[\"Content\",\"ContentImage\"],\"contentType\":[\"Resource\"],\"status\":[\"Live\"],\"compatibilityLevel\":{\"min\":1,\"max\":4},\"channel\":{\"ne\":[\"0124433024890224640\",\"0124446042259128320\",\"0124487522476933120\",\"0125840271570288640\",\"0124453662635048969\"]},\"framework\":{},\"mimeType\":{},\"resourceType\":{}},\"sort\":{\"me_averageRating\":\"desc\"},\"type\":\"content\",\"topn\":[{\"identifier\":\"do_31251340904692121624142\"},{\"identifier\":\"do_31257347965634969621957\"},{\"identifier\":\"do_312580373603835904112177\"},{\"identifier\":\"do_312580362281156608111821\"},{\"identifier\":\"do_312580363689975808211635\"}]},\"cdata\":[{\"id\":\"prod.diksha.app\",\"type\":\"appId\"}],\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":true},\"type\":\"events\",\"syncts\":1551977394569,\"@timestamp\":\"2019-03-07T16:49:54.569Z\",\"ts\":\"2019-03-07T16:49:54.372+0000\",\"devicedata\":{\"country\":\"India\",\"city\":\"Coimbatore\",\"countrycode\":\"IN\",\"state\":\"Tamil Nadu\",\"statecode\":\"TN\"}}";
-
-    /**
-     *
-     */
-    public static final String VALID_DIALCODETYPE ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":[{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":76854768347,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"}]}";
-
-    /**
-     * When array of object is having in the dialcodedata and which is having invalid property in one of the object in the array. ex: generatedon
-     */
-    public static final String INVALID_DIALCODETYPE_CASE_1 ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":[{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":43568797685,\"publishedon\":508978490534,\"status\":\"\",\"objecttype\":\"\"},{\"channel\":\"\",\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":\"43568797685\",\"publishedon\":508978490534,\"status\":true,\"objecttype\":\"\"}]}";
-
-    /**
-     * When dialcodedata object is having invalid properties
-     */
-    public static final String INVALID_DIALCODETYPE_CASE_2 ="{\"eid\":\"SEARCH\",\"ets\":1551332806642,\"ver\":\"3.0\",\"mid\":\"LP.1551332806642.922fb715-b4ed-421c-bae4-5ec48fd4d255\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"in.ekstep\",\"pdata\":{\"id\":\"staging.ntp.learning.platform\",\"pid\":\"search-service\",\"ver\":\"1.0\"},\"env\":\"search\"},\"edata\":{\"size\":259168,\"query\":\"\",\"filters\":{},\"sort\":{},\"type\":\"all\",\"topn\":[{\"identifier\":\"mh_fm_1_topic_science_usefulandharmfulmicrobes_harmfulmicroorganismsclostridiumandothers\"},{\"identifier\":\"mh_fm_1_topic_science_classificationofplants_subkingdomangiosperms\"},{\"identifier\":\"do_21254796078943436811896\"},{\"identifier\":\"do_21253024920136089612518\"},{\"identifier\":\"do_2125457233644748801997\"}]},\"flags\":{\"tv_processed\":true,\"dd_processed\":true,\"device_location_retrieved\":false},\"type\":\"events\",\"syncts\":1551341370427,\"@timestamp\":\"2019-02-28T08:09:30.427Z\",\"ts\":\"2019-02-28T05:46:46.642+0000\",\"devicedata\":{\"statecustomcode\":\"\",\"country\":\"\",\"city\":\"\",\"countrycode\":\"\",\"state\":\"\",\"statecode\":\"\",\"districtcustom\":\"\",\"statecustomname\":\"\"},\"dialcodedata\":{\"channel\":1234534534,\"batchcode\":\"\",\"publisher\":\"\",\"generatedon\":\"43568797685\",\"publishedon\":508978490534,\"status\":true,\"objecttype\":\"\"}}";
-
-    /**
-     *
-     */
-    public static final String VALID_LOG_EVENT = "{\"eid\":\"LOG\",\"ets\":1561717784892,\"ver\":\"3.0\",\"mid\":\"LP.1561717784892.1345e38d-f799-403f-93d9-1db71011627f\",\"actor\":{\"id\":\"org.ekstep.learning.platform\",\"type\":\"System\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.app\",\"pid\":\"learning-service\",\"ver\":\"1.0\"},\"env\":\"content\",\"did\":\"beea4401850780ee685120597a6428ab14ff47a2\"},\"edata\":{\"level\":\"INFO\",\"type\":\"api_access\",\"message\":\"\",\"params\":[{\"duration\":5},{\"protocol\":\"HTTP/1.1\"},{\"size\":10350},{\"method\":\"GET\"},{\"rid\":\"ekstep.content.find\"},{\"uip\":\"11.4.0.37\"},{\"url\":\"/learning-service/content/v3/read/do_31274885743485747215761\"},{\"status\":200}]},\"syncts\":1561717784892,\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"@timestamp\":\"2019-06-28T10:29:44.892Z\"}";
-
-    public static final String VALID_USER_DECLARED_AND_DERIVED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":\"Karnataka\",\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":\"Bangalore\",\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}";
-
-    public static final String INVALID_USER_DECLARED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":\"Karnataka\",\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":12,\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}";
-
-    public static final String IVALID_DERIVED_LOCATION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"393407b1-66b1-4c86-9080-b2bce9842886\"},\"eid\":\"INTERACT\",\"edata\":{\"id\":\"ContentDetail\",\"pageid\":\"ContentDetail\",\"type\":\"TOUCH\",\"subtype\":\"ContentDownload-Initiate\"},\"ver\":\"3.0\",\"ets\":1.54157454518E12,\"context\":{\"pdata\":{\"ver\":\"2.1.8\",\"pid\":\"sunbird.app\",\"id\":\"prod.diksha.app\"},\"channel\":\"0123221617357783046602\",\"env\":\"sdk\",\"did\":\"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\"cdata\":[{\"type\":\"qr\",\"id\":\"K4KCXE\"},{\"type\":\"API\",\"id\":\"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"}],\"sid\":\"70ea93d0-e521-4030-934f-276e7194c225\"},\"mid\":\"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\"object\":{\"id\":\"do_31249561779090227216256\",\"type\":\"Content\",\"version\":\"\"},\"tags\":[],\"syncts\":1.539846605341E12,\"@timestamp\":\"2018-10-18T07:10:05.341Z\",\"derivedlocationdata\":{\"district\":\"Bengaluru\",\"state\":1234,\"from\":\"user-profile\"},\"devicedata\":{\"statecustomcode\":\"KA-Custom\",\"country\":\"India\",\"iso3166statecode\":\"IN-KA\",\"city\":\"Bangalore\",\"countrycode\":\"IN\",\"statecode\":\"KA\",\"devicespec\":{\"os\":\"Android 6.0\",\"make\":\"Motorola XT1706\"},\"districtcustom\":\"Banglore-Custom\",\"firstaccess\":1559484698000,\"uaspec\":{\"agent\":\"Mozilla\",\"ver\":\"5.0\",\"system\":\"iPad\",\"raw\":\"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\"platform\":\"AppleWebKit/531.21.10\"},\"state\":\"Karnataka\",\"statecustomname\":\"Karnatak-Custom\",\"userdeclared\":{\"district\":\"Bangalore\",\"state\":\"Karnataka\"}},\"flags\":{\"device_location_retrieved\":true,\"device_profile_retrieved\":true}}";
-
-    public static final String INVALID_JSON  = "{\"eid\":\"LOG\",\"ets\":1570817279146,\"ver\":\"3.0\",\"mid\":\"LOG:2a210813fac274656f194f9807ad8abf\",\"actor\":{\"id\":\"1\",\"type\":\"service\"},\"context\":{\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.content-service\",\"ver\":\"1.0\",\"pid\":\"sunbird-content-service\"},\"env\":\"content\",\"sid\":\"\",\"did\":\"1e465bed2138b8f2764d946492293e2b2f628789\",\"cdata\":[],\"rollup\":{}},\"object\":{\"id\":\"do_312776645813747712111638\",\"type\":\"content\",\"ver\":\"\",\"rollup\":{}},\"tags\":,\"edata\":{\"type\":\"api_access\",\"level\":\"INFO\",\"message\":\"successful\",\"params\":[{\"rid\":\"0ece8640-ec52-11e9-a3d6-697ee5684e40\"},{\"title\":\"Content read api\"},{\"category\":\"contentread\"},{\"url\":\"content/read\"},{\"method\":\"GET\"}]},\"syncts\":1570817431399,\"@timestamp\":\"2019-10-11T18:10:31.399Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\"}";
-
-}
-
diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java
deleted file mode 100644
index 6762301687..0000000000
--- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/SummaryEventsValidatorTaskTest.java
+++ /dev/null
@@ -1,148 +0,0 @@
-
-package org.ekstep.ep.samza.task;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.SummaryV1;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.util.Map;
-
-public class SummaryEventsValidatorTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.denorm.valid";
-    private static final String FAILED_TOPIC = "telemetry.failed";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private static final String TELEMETRY_SCHEMA_PATH = "schemas/telemetry";
-    private static final String SUMMARY_SCHEMA_PATH = "schemas/summary";
-    private static final String DEFAULT_SCHEMA_NAME = "envelope.json";
-    private static final String SEARCH_SCHEMA_NAME = "search.json";
-    private static final String LOG_SCHEMA_NAME = "log.json";
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private DruidEventsValidatorTask druidEventsValidatorTask;
-
-    @Before
-    public void setUp() throws Exception {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = Mockito.mock(TaskContext.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = Mockito.mock(Config.class);
-
-        when(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).thenReturn(SUCCESS_TOPIC);
-        when(configMock.get("output.failed.topic.name", FAILED_TOPIC)).thenReturn(FAILED_TOPIC);
-        when(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).thenReturn(MALFORMED_TOPIC);
-        when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
-        when(contextMock.getMetricsRegistry()).thenReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.denorm", new Partition(1)));
-        when(configMock.get("telemetry.schema.path", TELEMETRY_SCHEMA_PATH)).thenReturn(TELEMETRY_SCHEMA_PATH);
-        when(configMock.get("summary.schema.path", SUMMARY_SCHEMA_PATH)).thenReturn(SUMMARY_SCHEMA_PATH);
-        when(configMock.get("event.schema.file", DEFAULT_SCHEMA_NAME)).thenReturn(DEFAULT_SCHEMA_NAME);
-        when(configMock.get("search.schema.file",SEARCH_SCHEMA_NAME)).thenReturn(SEARCH_SCHEMA_NAME);
-        when(configMock.get("log.schema.file",LOG_SCHEMA_NAME)).thenReturn(LOG_SCHEMA_NAME);
-        druidEventsValidatorTask = new DruidEventsValidatorTask(configMock, contextMock);
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(SummaryV1.VALID_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfEventIsNotValid() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "dimensions")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfEventIsNotValid_CASE_1() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_EVENT_CASE_1);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "tags")));
-    }
-
-    @Test
-    // When other than `ME_WORKFLOW_SUMMARY` Event is passed then it should send that event to success topic
-    public void ShouldSendEventToSuccessTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(SummaryV1.INVALID_SUMMARY);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    // uaspec object should validate
-    public void ShouldValidateUaspecObject() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(SummaryV1.UASPEC_SUMMARY);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "platform")));
-    }
-
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateEvent(final boolean dv_processed, final String field) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                Map<String, Object> event;
-                Map<String, Boolean> flags;
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String message = (String) outgoingMessageEnvelope.getMessage();
-                event = new Gson().fromJson(message, Map.class);
-                flags = new Gson().fromJson(event.get("flags").toString(), Map.class);
-                assertEquals(dv_processed, flags.get("dv_processed"));
-                if (!dv_processed) {
-                    assertEquals(true, event.get("metadata").toString().contains(field));
-                }
-                return true;
-            }
-        };
-    }
-}
-
diff --git a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java b/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java
deleted file mode 100644
index c795f77a4d..0000000000
--- a/data-pipeline/druid-events-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryEventsValidatorTaskTest.java
+++ /dev/null
@@ -1,256 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.TelemetryV3;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.util.Map;
-
-public class TelemetryEventsValidatorTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.denorm.valid";
-    private static final String FAILED_TOPIC = "telemetry.failed";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private static final String TELEMETRY_SCHEMA_PATH = "schemas/telemetry";
-    private static final String SUMMARY_SCHEMA_PATH = "schemas/summary";
-    private static final String DEFAULT_SCHEMA_NAME = "envelope.json";
-    private static final String SEARCH_SCHEMA_NAME = "search.json";
-    private static final String LOG_SCHEMA_NAME = "log.json";
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private DruidEventsValidatorTask druidEventsValidatorTask;
-
-    @Before
-    public void setUp() throws Exception {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = Mockito.mock(TaskContext.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = Mockito.mock(Config.class);
-
-        when(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).thenReturn(SUCCESS_TOPIC);
-        when(configMock.get("output.failed.topic.name", FAILED_TOPIC)).thenReturn(FAILED_TOPIC);
-        when(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).thenReturn(MALFORMED_TOPIC);
-        when(configMock.get("telemetry.schema.path", TELEMETRY_SCHEMA_PATH)).thenReturn(TELEMETRY_SCHEMA_PATH);
-        when(configMock.get("summary.schema.path", SUMMARY_SCHEMA_PATH)).thenReturn(SUMMARY_SCHEMA_PATH);
-        when(configMock.get("event.schema.file", DEFAULT_SCHEMA_NAME)).thenReturn(DEFAULT_SCHEMA_NAME);
-        when(configMock.get("search.schema.file",SEARCH_SCHEMA_NAME)).thenReturn(SEARCH_SCHEMA_NAME);
-        when(configMock.get("log.schema.file",LOG_SCHEMA_NAME)).thenReturn(LOG_SCHEMA_NAME);
-        when(metricsRegistry.newCounter(anyString(), anyString())).thenReturn(counter);
-        when(contextMock.getMetricsRegistry()).thenReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.denorm", new Partition(1)));
-        druidEventsValidatorTask = new DruidEventsValidatorTask(configMock, contextMock);
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfEventIsNotValid() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "contentdata/pkgversion")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE1() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_1);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "devicedata/firstaccess")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE2() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_2);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "devicedata/uaspec")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidDeviceData_CASE3() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_3);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "devicedata/country")));
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicIfInvalidDeviceData_CASE4() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DEVICEDATA_CASE_4);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidContentData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_CONTENTDATA);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "contentdata/language")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidDialCodeData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODEDATA);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "dialcodedata/objecttype")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicIfInvalidUserData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_USERDATA);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "userdata/gradelist")));
-    }
-
-    @Test
-    // Case sensitive dialcode keyword validation
-    public void shouldSendEventToFaildTopicIfInvalidDialCodeKeywordAppears() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODE_KEY);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    // Should support for the both array and object type format for dialcodedata .
-    public void shouldSendToSuccessTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_DIALCODETYPE);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    // When array of object properties is having invalid type ex: generatedon.
-    public void shouldSendToFailedTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODETYPE_CASE_1);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "generatedon")));
-    }
-
-    @Test
-    // When dialcodedata type is object and having invalid  property type ex: channel
-    public void shouldSendEventToFailedTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_DIALCODETYPE_CASE_2);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "channel")));
-    }
-
-    @Test
-    public void shouldSendLogEventToSuccessTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_LOG_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicForValidUserDeclaredAndDerivedLocationData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.VALID_USER_DECLARED_AND_DERIVED_LOCATION_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(true, "")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicForInvalidDerivedLocationData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.IVALID_DERIVED_LOCATION_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "derivedlocationdata/state")));
-    }
-
-    @Test
-    public void shouldSendEventToFaildTopicForInvalidUserDeclaredLocationData() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_USER_DECLARED_LOCATION_EVENT);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        verify(collectorMock).send(argThat(validateEvent(false, "devicedata/userdeclared/district")));
-    }
-
-    /**
-     * When Json exception occurs then it should send the event to malformed topic
-     * @throws Exception
-     */
-    @Test
-    public void shouldAddInvalidEventToMalformedTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(TelemetryV3.INVALID_JSON);
-        druidEventsValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-    }
-
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateEvent(final boolean dv_processed, final String field) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                Map<String, Object> event;
-                Map<String, Boolean> flags;
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String message = (String) outgoingMessageEnvelope.getMessage();
-                event = new Gson().fromJson(message, Map.class);
-                flags = new Gson().fromJson(event.get("flags").toString(), Map.class);
-                assertEquals(dv_processed, flags.get("dv_processed"));
-                if (!dv_processed) {
-                    assertEquals(true, event.get("metadata").toString().contains(field));
-                }
-                return true;
-            }
-        };
-    }
-}
diff --git a/data-pipeline/ep-core/pom.xml b/data-pipeline/ep-core/pom.xml
deleted file mode 100644
index 6cacadafc6..0000000000
--- a/data-pipeline/ep-core/pom.xml
+++ /dev/null
@@ -1,164 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>jobs</artifactId>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <version>0.0.1</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>ep-core</artifactId>
-    <version>0.0.1</version>
-    <packaging>jar</packaging>
-    <name>ep-core</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-core_${scala.version}</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-yarn_${scala.version}</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-kafka_${scala.version}</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-kv_${scala.version}</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-kv-rocksdb_${scala.version}</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>postgresql</groupId>
-            <artifactId>postgresql</artifactId>
-            <version>9.1-901.jdbc4</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.opentable.components</groupId>
-            <artifactId>otj-pg-embedded</artifactId>
-            <version>0.13.3</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-		    <groupId>it.ozimov</groupId>
-		    <artifactId>embedded-redis</artifactId>
-		    <version>0.7.1</version>
-		    <scope>test</scope>
-		</dependency>
-        <dependency>
-            <groupId>com.squareup.okhttp3</groupId>
-            <artifactId>mockwebserver</artifactId>
-            <version>4.4.0</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.konghq</groupId>
-            <artifactId>unirest-java</artifactId>
-            <version>3.4.01</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                    <excludes>
-                        <exclude>org/ekstep/ep/samza/util/**/CassandraConnect.class</exclude>
-                        <exclude>org/ekstep/ep/samza/core/**/Logger.class</exclude>
-                    </excludes>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java
deleted file mode 100644
index 241bf2eb29..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseCacheUpdaterService.java
+++ /dev/null
@@ -1,87 +0,0 @@
-package org.ekstep.ep.samza.core;
-
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.exceptions.DriverException;
-import com.datastax.driver.core.querybuilder.Clause;
-import com.datastax.driver.core.querybuilder.Insert;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.util.CassandraConnect;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class BaseCacheUpdaterService {
-
-    private RedisConnect redisConnect;
-    private CassandraConnect cassandraConnect;
-    private Jedis connection;
-
-    public BaseCacheUpdaterService(RedisConnect redisConnect) {
-        this.redisConnect = redisConnect;
-        connection = redisConnect.getConnection();
-    }
-
-    public BaseCacheUpdaterService(CassandraConnect cassandraConnect){
-        this.cassandraConnect = cassandraConnect;
-    }
-
-    public BaseCacheUpdaterService(RedisConnect redisConnect, CassandraConnect cassandraConnect) {
-        this.redisConnect = redisConnect;
-        connection = redisConnect.getConnection();
-        this.cassandraConnect = cassandraConnect;
-    }
-
-    public void addToCache(String key, String value, int storeId) {
-        try {
-            connection.select(storeId);
-            if (key != null && !key.isEmpty() && null != value && !value.isEmpty()) {
-                connection.set(key, value);
-            }
-        } catch(JedisException ex) {
-            connection = redisConnect.getConnection(storeId);
-            if (null != value)
-                connection.set(key, value);
-        }
-    }
-
-    public String readFromCache(String key, int storeId) {
-        try {
-            connection.select(storeId);
-            return connection.get(key);
-        }
-        catch (JedisException ex) {
-            connection = redisConnect.getConnection(storeId);
-            return connection.get(key);
-        }
-    }
-
-    public void updateToCassandra(String keyspace, String table, Map<String, String> data) {
-        Insert query = QueryBuilder.insertInto(keyspace, table)
-                .values(new ArrayList<>(data.keySet()), new ArrayList<>(data.values()));
-        try {
-            cassandraConnect.upsert(query);
-        } catch(DriverException ex) {
-            cassandraConnect.reconnectCluster();
-            cassandraConnect.upsert(query);
-        }
-    }
-
-    public List<Row> readFromCassandra(String keyspace, String table, Clause clause) {
-        List<Row> rowSet = null;
-        String query = QueryBuilder.select().all()
-                .from(keyspace, table)
-                .where(clause)
-                .toString();
-        try {
-            rowSet = cassandraConnect.find(query);
-        } catch(DriverException ex) {
-            cassandraConnect.reconnectCluster();
-            rowSet = cassandraConnect.find(query);
-        }
-        return rowSet;
-    }
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java
deleted file mode 100644
index 24da8c58f0..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/BaseSink.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.ekstep.ep.samza.core;
-
-import java.util.Map;
-
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.task.MessageCollector;
-
-public class BaseSink {
-
-	private MessageCollector collector;
-	protected JobMetrics metrics;
-	
-	public BaseSink(MessageCollector collector) {
-		this.collector = collector;
-	}
-
-	public BaseSink(MessageCollector collector, JobMetrics metrics) {
-		this.collector = collector;
-		this.metrics = metrics;
-	}
-	
-	public void toTopic(String topic, String mid, String message) {
-		collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", topic), mid, message));
-	}
-	
-	public void toTopic(String topic, String mid, Map<String, Object> message) {
-		collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", topic), mid, message));
-	}
-
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java
deleted file mode 100644
index d71103af54..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/JobMetrics.java
+++ /dev/null
@@ -1,403 +0,0 @@
-package org.ekstep.ep.samza.core;
-
-import com.google.gson.Gson;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.Metric;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.metrics.MetricsRegistryMap;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.TaskContext;
-import org.joda.time.DateTime;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-
-public class JobMetrics {
-    private static Logger LOGGER = new Logger(JobMetrics.class);
-    private final String jobName;
-    private final Counter successMessageCount;
-    private final Counter failedMessageCount;
-    private final Counter skippedMessageCount;
-    private final Counter errorMessageCount;
-    private final Counter batchSuccessCount;
-    private final Counter batchErrorCount;
-    private final Counter primaryRouteSuccessCount;
-    private final Counter secondaryRouteSuccessCount;
-    private final Counter logRouteSuccessCount;
-    private final Counter errorRouteSuccessCount;
-    private final Counter auditRouteSuccessCount;
-    private final Counter assessRouteSuccessCount;
-    private final Counter shareEventRouteSuccessCount;
-    private final Counter cacheHitCount;
-    private final Counter cacheMissCount;
-    private final Counter cacheErrorCount;
-    private final Counter cacheEmptyValuesCount;
-    private final Counter processedMessageCount;
-    private final Counter unprocessedMessageCount;
-    private final Counter dbHitCount;
-    private final Counter userCacheHitCount;
-    private final Counter expiredEventCount;
-    private final Counter duplicateEventCount;
-    private final Counter deviceDBUpdateCount;
-    private final Counter deviceCacheUpdateCount;
-    private final Counter userDeclaredHitCount;
-    private final Counter ipLocationHitCount;
-    
-    private final Counter dbInsertCount;
-    private final Counter dbUpdateCount;
-    private final Counter dialCodesCount;
-    private final Counter dialCodesFromApiCount;
-    private final Counter dialCodesFromCacheCount;
-
-    private TaskContext context;
-    private int partition;
-
-    private Map<String, Counter> metricCounterMap = new HashMap<>();
-
-    public JobMetrics(TaskContext context) {
-        this(context, null);
-    }
-
-    public JobMetrics(TaskContext context, String jName) {
-        MetricsRegistry metricsRegistry = context.getMetricsRegistry();
-        successMessageCount = metricsRegistry.newCounter(getClass().getName(), "success-message-count");
-        failedMessageCount = metricsRegistry.newCounter(getClass().getName(), "failed-message-count");
-        skippedMessageCount = metricsRegistry.newCounter(getClass().getName(), "skipped-message-count");
-        errorMessageCount = metricsRegistry.newCounter(getClass().getName(), "error-message-count");
-        batchSuccessCount = metricsRegistry.newCounter(getClass().getName(), "batch-success-count");
-        batchErrorCount = metricsRegistry.newCounter(getClass().getName(), "batch-error-count");
-        primaryRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "primary-route-success-count");
-        secondaryRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "secondary-route-success-count");
-        logRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "log-route-success-count");
-        errorRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "error-route-success-count");;
-        auditRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "audit-route-success-count");
-        assessRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "assess-route-success-count");
-        shareEventRouteSuccessCount = metricsRegistry.newCounter(getClass().getName(), "share-route-success-count");
-        cacheHitCount = metricsRegistry.newCounter(getClass().getName(), "cache-hit-count");
-        cacheMissCount = metricsRegistry.newCounter(getClass().getName(), "cache-miss-count");
-        cacheEmptyValuesCount = metricsRegistry.newCounter(getClass().getName(), "cache-empty-values-count");
-        cacheErrorCount = metricsRegistry.newCounter(getClass().getName(), "cache-error-count");
-        processedMessageCount = metricsRegistry.newCounter(getClass().getName(), "processed-message-count");
-        unprocessedMessageCount = metricsRegistry.newCounter(getClass().getName(), "unprocessed-message-count");
-        dbHitCount = metricsRegistry.newCounter(getClass().getName(), "db-hit-count");
-        userCacheHitCount = metricsRegistry.newCounter(getClass().getName(), "user-cache-hit-count");
-        expiredEventCount = metricsRegistry.newCounter(getClass().getName(), "expired-event-count");
-        duplicateEventCount = metricsRegistry.newCounter(getClass().getName(), "duplicate-event-count");
-        deviceDBUpdateCount = metricsRegistry.newCounter(getClass().getName(), "device-db-update-count");
-        deviceCacheUpdateCount = metricsRegistry.newCounter(getClass().getName(), "device-cache-update-count");
-        userDeclaredHitCount = metricsRegistry.newCounter(getClass().getName(), "user-declared-hit-count");
-        ipLocationHitCount = metricsRegistry.newCounter(getClass().getName(), "ip-location-hit-count");
-        dbInsertCount = metricsRegistry.newCounter(getClass().getName(), "db-insert-count");
-        dbUpdateCount = metricsRegistry.newCounter(getClass().getName(), "db-update-count");
-        dialCodesCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-count");
-        dialCodesFromApiCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-from-api-count");
-        dialCodesFromCacheCount = metricsRegistry.newCounter(getClass().getName(),"dial-codes-from-cache-count");
-
-        metricCounterMap.put("success-message-count", successMessageCount);
-        metricCounterMap.put("failed-message-count", failedMessageCount);
-        metricCounterMap.put("skipped-message-count", skippedMessageCount);
-        metricCounterMap.put("error-message-count", errorMessageCount);
-        metricCounterMap.put("batch-success-count", batchSuccessCount);
-        metricCounterMap.put("batch-error-count", batchErrorCount);
-        metricCounterMap.put("primary-route-success-count", primaryRouteSuccessCount);
-        metricCounterMap.put("secondary-route-success-count", secondaryRouteSuccessCount);
-        metricCounterMap.put("log-route-success-count", logRouteSuccessCount);
-        metricCounterMap.put("error-route-success-count", errorRouteSuccessCount);
-        metricCounterMap.put("audit-route-success-count", auditRouteSuccessCount);
-        metricCounterMap.put("assess-route-success-count", assessRouteSuccessCount);
-        metricCounterMap.put("cache-hit-count", cacheHitCount);
-        metricCounterMap.put("cache-miss-count", cacheMissCount);
-        metricCounterMap.put("cache-empty-values-count", cacheEmptyValuesCount);
-        metricCounterMap.put("cache-error-count", cacheErrorCount);
-        metricCounterMap.put("processed-message-count", processedMessageCount);
-        metricCounterMap.put("unprocessed-message-count", unprocessedMessageCount);
-        metricCounterMap.put("user-cache-hit-count", userCacheHitCount);
-        metricCounterMap.put("db-hit-count", dbHitCount);
-        metricCounterMap.put("expired-event-count", expiredEventCount);
-        metricCounterMap.put("duplicate-event-count", duplicateEventCount);
-        metricCounterMap.put("device-db-update-count", deviceDBUpdateCount);
-        metricCounterMap.put("device-cache-update-count", deviceCacheUpdateCount);
-        metricCounterMap.put("user-declared-hit-count", userDeclaredHitCount);
-        metricCounterMap.put("ip-location-hit-count", ipLocationHitCount);
-        metricCounterMap.put("db-insert-count", dbInsertCount);
-        metricCounterMap.put("db-update-count", dbUpdateCount);
-        metricCounterMap.put("dialcodes-count", dialCodesCount);
-        metricCounterMap.put("dialcodes-api-hit", dialCodesFromApiCount);
-        metricCounterMap.put("dialcodes-cache-hit", dialCodesFromCacheCount);
-        jobName = jName;
-        this.context = context;
-    }
-
-    public void clear() {
-        successMessageCount.clear();
-        failedMessageCount.clear();
-        skippedMessageCount.clear();
-        errorMessageCount.clear();
-        batchSuccessCount.clear();
-        batchErrorCount.clear();
-        cacheEmptyValuesCount.clear();
-        cacheHitCount.clear();
-        cacheMissCount.clear();
-        cacheErrorCount.clear();
-        processedMessageCount.clear();
-        unprocessedMessageCount.clear();
-        dbHitCount.clear();
-        userCacheHitCount.clear();
-        expiredEventCount.clear();
-        duplicateEventCount.clear();
-        deviceDBUpdateCount.clear();
-        deviceCacheUpdateCount.clear();
-        userDeclaredHitCount.clear();
-        ipLocationHitCount.clear();
-        primaryRouteSuccessCount.clear();
-        secondaryRouteSuccessCount.clear();
-        logRouteSuccessCount.clear();
-        errorRouteSuccessCount.clear();
-        auditRouteSuccessCount.clear();
-        assessRouteSuccessCount.clear();
-        shareEventRouteSuccessCount.clear();
-        dbInsertCount.clear();
-        dbUpdateCount.clear();
-        dialCodesCount.clear();
-        dialCodesFromApiCount.clear();
-        dialCodesFromCacheCount.clear();
-    }
-
-    public void incSuccessCounter() {
-        successMessageCount.inc();
-    }
-
-    public void deviceDBUpdateSuccess() {
-        deviceDBUpdateCount.inc();
-    }
-
-    public void deviceCacheUpdateSuccess() {
-        deviceCacheUpdateCount.inc();
-    }
-
-    public void incFailedCounter() {
-        failedMessageCount.inc();
-    }
-
-    public void incSkippedCounter() {
-        skippedMessageCount.inc();
-    }
-
-    public void incErrorCounter() {
-        errorMessageCount.inc();
-    }
-
-    public void incBatchSuccessCounter() {
-        batchSuccessCount.inc();
-    }
-
-    public void incBatchErrorCounter() {
-        batchErrorCount.inc();
-    }
-
-    public void incPrimaryRouteSuccessCounter() {
-        primaryRouteSuccessCount.inc();
-    }
-
-    public void incSecondaryRouteSuccessCounter() {
-        secondaryRouteSuccessCount.inc();
-    }
-
-    public void incLogRouteSuccessCounter() {
-        logRouteSuccessCount.inc();
-    }
-
-    public void incErrorRouteSuccessCounter() {
-        errorRouteSuccessCount.inc();
-    }
-    
-    public void incAuditRouteSuccessCounter() {
-        auditRouteSuccessCount.inc();
-    }
-    
-    public void incAssessRouteSuccessCounter() {
-      assessRouteSuccessCount.inc();
-  }
-
-    public void incShareEventRouteSuccessCounter() {
-        shareEventRouteSuccessCount.inc();
-    }
-
-    public void incDuplicateCounter() {
-        duplicateEventCount.inc();
-    }
-
-    public void incCacheHitCounter() {
-        cacheHitCount.inc();
-    }
-
-    public void incCacheMissCounter() {
-        cacheMissCount.inc();
-    }
-
-    public void incCacheErrorCounter() {
-        cacheErrorCount.inc();
-    }
-
-    public void incEmptyCacheValueCounter() {
-        cacheEmptyValuesCount.inc();
-    }
-
-    public void incProcessedMessageCount() {
-        processedMessageCount.inc();
-    }
-
-    public void incUnprocessedMessageCount() {
-        unprocessedMessageCount.inc();
-    }
-
-    public void incDBHitCount() {
-        dbHitCount.inc();
-    }
-
-    public void incUserCacheHitCount() {
-        userCacheHitCount.inc();
-    }
-
-    public void incExpiredEventCount() {
-        expiredEventCount.inc();
-    }
-
-    public void incUserDeclaredHitCount() {
-        userDeclaredHitCount.inc();
-    }
-
-    public void incIpLocationHitCount() {
-        ipLocationHitCount.inc();
-    }
-
-    public void incDBInsertCount() {
-        dbInsertCount.inc();
-    }
-    
-    public void incDBUpdateCount() {
-        dbUpdateCount.inc();
-    }
-
-    public void incDialCodesCount() { dialCodesCount.inc(); }
-
-    public void incDialCodesFromApiCount() { dialCodesFromApiCount.inc(); }
-
-    public void incDialCodesFromCacheCount() { dialCodesFromCacheCount.inc(); }
-
-    public long consumerLag(Map<String, ConcurrentHashMap<String, Metric>> containerMetricsRegistry) {
-        long consumerLag = 0;
-        try {
-            for (SystemStreamPartition sysPartition : context.getSystemStreamPartitions()) {
-                long highWatermarkOffsetForPartition =
-                        Long.valueOf(containerMetricsRegistry.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics")
-                                .get(getSamzaMetricKey(sysPartition, "high-watermark")).toString());
-                long checkpointedOffsetForPartition = Long.valueOf(containerMetricsRegistry.get("org.apache.samza.checkpoint.OffsetManagerMetrics")
-                        .get(getSamzaMetricKey(sysPartition, "checkpointed-offset")).toString());
-                consumerLag += highWatermarkOffsetForPartition - checkpointedOffsetForPartition;
-                this.partition = sysPartition.getPartition().getPartitionId();
-            }
-
-        } catch (Exception e) {
-            LOGGER.error(null, "EXCEPTION. WHEN COMPUTING CONSUMER LAG METRIC", e);
-        }
-        return consumerLag;
-    }
-
-    private String getSamzaMetricKey(SystemStreamPartition partition, String samzaMetricName) {
-        return String.format("%s-%s-%s-%s",
-                partition.getSystem(), partition.getStream(), partition.getPartition().getPartitionId(), samzaMetricName);
-    }
-
-    public String collect() {
-        Map<String, Object> metricsEvent = new HashMap<>();
-        metricsEvent.put("job-name", jobName);
-        metricsEvent.put("partition", partition);
-        metricsEvent.put("success-message-count", successMessageCount.getCount());
-        metricsEvent.put("failed-message-count", failedMessageCount.getCount());
-        metricsEvent.put("error-message-count", errorMessageCount.getCount());
-        metricsEvent.put("batch-success-count", batchSuccessCount.getCount());
-        metricsEvent.put("batch-error-count", batchErrorCount.getCount());
-        metricsEvent.put("primary-route-success-count", primaryRouteSuccessCount.getCount());
-        metricsEvent.put("secondary-route-success-count", secondaryRouteSuccessCount.getCount());
-        metricsEvent.put("log-route-success-count", logRouteSuccessCount.getCount());
-        metricsEvent.put("error-route-success-count", errorRouteSuccessCount.getCount());
-        metricsEvent.put("skipped-message-count", skippedMessageCount.getCount());
-        metricsEvent.put("cache-hit-count", cacheHitCount.getCount());
-        metricsEvent.put("cache-miss-count", cacheMissCount.getCount());
-        metricsEvent.put("cache-error-count", cacheErrorCount.getCount());
-        metricsEvent.put("cache-empty-values-count", cacheEmptyValuesCount.getCount());
-        metricsEvent.put("processed-message-count", processedMessageCount.getCount());
-        metricsEvent.put("unprocessed-message-count", unprocessedMessageCount.getCount());
-        metricsEvent.put("db-hit-count", dbHitCount.getCount());
-        metricsEvent.put("user-cache-hit-count", userCacheHitCount.getCount());
-        metricsEvent.put("expired-event-count", expiredEventCount.getCount());
-        metricsEvent.put("duplicate-event-count", duplicateEventCount.getCount());
-
-        metricsEvent.put("device-db-update-count", deviceDBUpdateCount.getCount());
-        metricsEvent.put("device-cache-update-count", deviceCacheUpdateCount.getCount());
-        metricsEvent.put("user-declared-hit-count", userDeclaredHitCount.getCount());
-        metricsEvent.put("ip-location-hit-count", ipLocationHitCount.getCount());
-        metricsEvent.put("audit-route-success-count", auditRouteSuccessCount.getCount());
-        metricsEvent.put("share-route-success-count", shareEventRouteSuccessCount.getCount());
-        metricsEvent.put("db-insert-count", dbInsertCount.getCount());
-        metricsEvent.put("db-update-count", dbUpdateCount.getCount());
-        metricsEvent.put("dialcodes-count", dialCodesCount.getCount());
-        metricsEvent.put("dialcodes-api-hit", dialCodesCount.getCount());
-        metricsEvent.put("dialcodes-cache-hit", dialCodesCount.getCount());
-
-        metricsEvent.put("consumer-lag",
-                consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics()));
-        metricsEvent.put("metricts", new DateTime().getMillis());
-        return new Gson().toJson(metricsEvent);
-    }
-
-    public String collect(List<String> metrics) {
-        Map<String, Object> metricsEvent = new HashMap<>();
-        metricsEvent.put("job-name", jobName);
-        metricsEvent.put("partition", partition);
-        metricsEvent.put("consumer-lag",
-                consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics()));
-        metricsEvent.put("metricts", new DateTime().getMillis());
-        for (String metric: metrics) {
-            metricsEvent.put(metric, metricCounterMap.get(metric).getCount());
-        }
-        return new Gson().toJson(metricsEvent);
-    }
-
-    public String generateMetrics(List<String> metrics) {
-        Map<String, Object> metricsEvent = new HashMap<>();
-        metricsEvent.put("system", "samza");
-        metricsEvent.put("subsystem", "pipeline-metrics");
-        metricsEvent.put("metricts", new DateTime().getMillis());
-        List<Map<String, Object>> dimsList = new ArrayList<>();
-        dimsList.add(createMap("job-name", jobName));
-        dimsList.add(createMap("partition", partition));
-
-        List<Map<String, Object>> metricsList = new ArrayList<>();
-        for (String metric: metrics) {
-            Map<String, Object> mMap = new HashMap<>();
-            mMap.put("id", metric);
-            mMap.put("value", metricCounterMap.get(metric).getCount());
-            metricsList.add(mMap);
-        }
-
-        metricsList.add(createMap("consumer-lag",
-                consumerLag(((MetricsRegistryMap) context.getSamzaContainerContext().metricsRegistry).metrics())));
-
-        metricsEvent.put("dimensions", dimsList);
-        metricsEvent.put("metrics", metricsList);
-        return new Gson().toJson(metricsEvent);
-    }
-
-    private Map<String, Object> createMap(String id, Object value) {
-        Map<String, Object> map = new HashMap<>();
-        map.put("id", id);
-        map.put("value", value);
-        return map;
-    }
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java
deleted file mode 100644
index 53aa3bfea5..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/core/Logger.java
+++ /dev/null
@@ -1,63 +0,0 @@
-package org.ekstep.ep.samza.core;
-
-import org.slf4j.LoggerFactory;
-
-import static java.text.MessageFormat.format;
-
-public class Logger {
-
-    private final org.slf4j.Logger logger;
-
-    public Logger(Class<?> clazz) {
-        logger = LoggerFactory.getLogger(clazz);
-    }
-
-    public void trace(String eventId, String s) {
-        logger.trace(logPrependedWithMetadata(eventId, s));
-    }
-
-    public void trace(String eventId, String s, Object o) {
-        logger.trace(logPrependedWithMetadata(eventId, s), o);
-    }
-
-    public void debug(String eventId, String s) {
-        logger.debug(logPrependedWithMetadata(eventId, s));
-    }
-
-    public void debug(String eventId, String s, Object o) {
-        logger.debug(logPrependedWithMetadata(eventId, s), o);
-    }
-
-    public void info(String eventId, String s) {
-        logger.info(logPrependedWithMetadata(eventId, s));
-    }
-
-    public void info(String eventId, String s, Object o) {
-        logger.info(logPrependedWithMetadata(eventId, s), o);
-    }
-
-    public void warn(String eventId, String s) {
-        logger.warn(logPrependedWithMetadata(eventId, s));
-    }
-
-    public void warn(String eventId, String s, Object o) {
-        logger.warn(logPrependedWithMetadata(eventId, s), o);
-    }
-
-    public void error(String eventId, String s) {
-        logger.error(logPrependedWithMetadata(eventId, s));
-    }
-
-    public void error(String eventId, String s, Object o) {
-        logger.error(logPrependedWithMetadata(eventId, s), o);
-    }
-
-    public void error(String eventId, String s, Throwable throwable) {
-        logger.error(logPrependedWithMetadata(eventId, s), throwable);
-        throwable.printStackTrace();
-    }
-
-    private String logPrependedWithMetadata(String eventId, String s) {
-        return eventId == null ? s : format("{0} {1}", eventId, s);
-    }
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java
deleted file mode 100644
index a72266f032..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/task/BaseSamzaTask.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.task.InitableTask;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.StreamTask;
-import org.apache.samza.task.TaskCoordinator;
-import org.apache.samza.task.WindowableTask;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-abstract class BaseSamzaTask implements StreamTask, InitableTask, WindowableTask {
-	
-	protected JobMetrics metrics;
-	private String metricsTopic;
-	private List<String> metricsList;
-	private String prometheusMetricsTopic;
-	private List<String> defaultPipelineMetrics =
-			new ArrayList<>(Arrays.asList("success-message-count", "failed-message-count", "error-message-count", "skipped-message-count"));
-	
-	public BaseSamzaTask() {
-		
-	}
-	
-	public void initTask(Config config, JobMetrics metrics) {
-		this.metrics = metrics;
-		this.metricsTopic = config.get("output.metrics.topic.name", "telemetry.pipeline_metrics");
-		this.prometheusMetricsTopic = config.get("output.prometheus.metrics.topic.name", "telemetry.metrics");
-		this.metricsList = config.getList("pipeline.metrics.list", defaultPipelineMetrics);
-	}
-	
-
-	@Override
-	public void window(MessageCollector collector, TaskCoordinator coordinator) throws Exception {
-
-		String mEvent = metrics.collect(metricsList);
-		String prometheusMetricEvent = metrics.generateMetrics(metricsList);
-		collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", metricsTopic), mEvent));
-		collector.send(new OutgoingMessageEnvelope(new SystemStream("kafka", prometheusMetricsTopic), prometheusMetricEvent));
-		this.metrics.clear();
-	}
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java
deleted file mode 100644
index 29bb94c6c5..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/CassandraConnect.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.datastax.driver.core.*;
-
-import java.util.List;
-
-import com.datastax.driver.core.querybuilder.Insert;
-import org.apache.samza.config.Config;
-
-public class CassandraConnect {
-    private Session session;
-    private Config config;
-    private List<String> clusterHosts;
-    private int port;
-
-    public CassandraConnect(Config config) {
-        this.config = config;
-        String host = config.get("cassandra.host", "127.0.0.1");
-        Integer port = config.getInt("cassandra.port", 9042);
-        Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build();
-        this.session = cluster.connect();
-    }
-
-    public CassandraConnect(String host, Integer port) {
-        Cluster cluster = Cluster.builder().addContactPoints(host).withPort(port).build();
-        this.session = cluster.connect();
-    }
-
-    public CassandraConnect(List<String> hosts, Integer port) {
-        this.clusterHosts = hosts;
-        this.port = port;
-        Cluster.Builder builder = Cluster.builder();
-        for (String host : hosts) {
-            builder.addContactPoint(host);
-        }
-        // TODO: We need to check if the consistency level can be downgraded to ONE
-        Cluster cluster = builder.withPort(port).withQueryOptions(new QueryOptions()
-                .setConsistencyLevel(ConsistencyLevel.QUORUM)).build();
-        this.session = cluster.connect();
-    }
-
-    public Row findOne(String query) {
-        ResultSet rs = session.execute(query);
-        return rs.one();
-    }
-
-    public List<Row> find(String query) {
-        ResultSet rs = session.execute(query);
-        return rs.all();
-    }
-
-    public boolean upsert(String query) {
-        ResultSet rs = session.execute(query);
-        return rs.wasApplied();
-    }
-
-    public boolean upsert(Insert query) {
-        ResultSet rs = session.execute(query);
-        return rs.wasApplied();
-    }
-
-    public UserType getUDTType(String keyspace, String typeName) {
-        return session.getCluster().getMetadata().getKeyspace(keyspace).getUserType(typeName);
-    }
-
-    public void reconnect() {
-        this.session.close();
-        String host = config.get("cassandra.host", "127.0.0.1");
-        Integer port = config.getInt("cassandra.port", 9042);
-        Cluster cluster = Cluster.builder().addContactPoint(host).withPort(port).build();
-        this.session = cluster.connect();
-    }
-
-    public void reconnectCluster() {
-        this.session.close();
-        Cluster.Builder builder = Cluster.builder();
-        for (String host : clusterHosts) {
-            builder.addContactPoint(host);
-        }
-        Cluster cluster = builder.withPort(port).withQueryOptions(new QueryOptions()
-                .setConsistencyLevel(ConsistencyLevel.QUORUM)).build();
-        this.session = cluster.connect();
-    }
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java
deleted file mode 100644
index 8fb88478cf..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/DeDupEngine.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-
-public class DeDupEngine {
-
-    private RedisConnect redisConnect;
-    private Jedis redisConnection;
-    private int store;
-    private int expirySeconds;
-
-    public DeDupEngine(RedisConnect redisConnect, int store, int expirySeconds) {
-      
-        this.redisConnect = redisConnect;
-        this.redisConnection = redisConnect.getConnection();
-        this.store = store;
-        this.redisConnection.select(store);
-        this.expirySeconds = expirySeconds;
-    }
-
-    public boolean isUniqueEvent(String checksum) throws JedisException {
-
-        boolean unique = false;
-        try {
-            unique = !redisConnection.exists(checksum);
-        } catch (JedisException ex) {
-            this.redisConnection.close();
-            this.redisConnection = redisConnect.getConnection(store, 10000);
-            unique = !redisConnection.exists(checksum);
-        }
-        return unique;
-    }
-
-    public void storeChecksum(String checksum) throws JedisException {
-
-        try {
-            redisConnection.setex(checksum, expirySeconds, "");
-        } catch (JedisException ex) {
-            this.redisConnection.close();
-            this.redisConnection = redisConnect.getConnection(10000);
-            this.redisConnection.select(store);
-            redisConnection.setex(checksum, expirySeconds, "");
-        }
-    }
-
-    public Jedis getRedisConnection() {
-        return redisConnection;
-    }
-
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java
deleted file mode 100644
index 36774328b7..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/PostgresConnect.java
+++ /dev/null
@@ -1,70 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.apache.samza.config.Config;
-import org.postgresql.ds.PGPoolingDataSource;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-public class PostgresConnect {
-    private Config config;
-    private Connection connection;
-    private Statement statement;
-    private String user;
-    private String password;
-    private String db;
-    private String host;
-    private Integer port;
-    private Integer maxConnections;
-    private PGPoolingDataSource source;
-
-    public PostgresConnect(Config config) throws Exception {
-        this.config = config;
-        user = config.get("postgres.user");
-        password = config.get("sensitive.postgres.password");
-        db = config.get("postgres.db");
-        host = config.get("postgres.host","127.0.0.1");
-        port = config.getInt("postgres.port", 5432);
-        maxConnections = config.getInt("postgres.maxConnections", 2);
-        buildPoolConfig();
-        connection = source.getConnection();
-        statement = connection.createStatement();
-    }
-
-    public void buildPoolConfig() throws Exception {
-        Class.forName("org.postgresql.Driver");
-        source = new PGPoolingDataSource();
-        source.setServerName(host);
-        source.setPortNumber(port);
-        source.setUser(user);
-        source.setPassword(password);
-        source.setDatabaseName(db);
-        source.setMaxConnections(maxConnections);
-    }
-
-    public Connection getConnection() {
-        return this.connection;
-    }
-
-    public boolean execute(String query) throws Exception {
-        try {
-            return statement.execute(query);
-        } catch (SQLException ex) {
-            resetConnection();
-            return statement.execute(query);
-        }
-    }
-
-    public Connection resetConnection() throws Exception {
-        closeConnection();
-        buildPoolConfig();
-        connection = source.getConnection();
-        statement = connection.createStatement();
-        return connection;
-    }
-
-    public void closeConnection() throws Exception {
-        connection.close();
-        source.close();
-    }
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java
deleted file mode 100644
index 259bbb14e2..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RedisConnect.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.apache.samza.config.Config;
-import redis.clients.jedis.Jedis;
-
-
-public class RedisConnect {
-
-    private Config config;
-
-    public RedisConnect(Config config) {
-        this.config = config;
-    }
-
-    private Jedis getConnection(long backoffTimeInMillis) {
-        String redisHost = config.get("redis.host", "localhost");
-        Integer redisPort = config.getInt("redis.port", 6379);
-        if(backoffTimeInMillis > 0) {
-            try {
-                Thread.sleep(backoffTimeInMillis);
-            } catch (InterruptedException e) {
-                e.printStackTrace();
-            }
-        }
-        return new Jedis(redisHost, redisPort, 30000);
-    }
-    
-    public Jedis getConnection(int db, long backoffTimeInMillis) {
-        
-        Jedis jedis = getConnection(backoffTimeInMillis);
-        jedis.select(db);
-        return jedis;
-    }
-    
-    public Jedis getConnection(int db) {
-      
-        Jedis jedis = getConnection(db, 0);
-        jedis.select(db);
-        return jedis;
-    }
-    
-    public Jedis getConnection() {
-      
-        return getConnection(0);
-    }
-    
-}
diff --git a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java b/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java
deleted file mode 100644
index 89a35d8948..0000000000
--- a/data-pipeline/ep-core/src/main/java/org/ekstep/ep/samza/util/RestUtil.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-
-import kong.unirest.GetRequest;
-import kong.unirest.Unirest;
-import kong.unirest.UnirestException;
-
-import java.util.Map;
-
-public class RestUtil {
-
-    /**
-     * @param apiURL         - API Url
-     * @param requestHeaders - Map<String, String> Request header parameters
-     * @return - Unirest.ResponseBody
-     * @throws UnirestException
-     */
-    public String get(String apiURL, Map<String, String> requestHeaders) throws UnirestException {
-        GetRequest request = Unirest.get(apiURL);
-        request.headers(requestHeaders);
-        return request.asString().getBody();
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java
deleted file mode 100644
index 39332ce2f6..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/fixtures/MetricsFixture.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.Metric;
-
-import java.lang.reflect.Type;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class MetricsFixture {
-
-	public static final String METRIC_EVENT_STREAM1 = "{\n" +
-			"  \"org.apache.samza.system.kafka.KafkaSystemConsumerMetrics\": {\n" +
-			"    \"kafka-inputtopic-0-high-watermark\": {\n" +
-			"      \"name\": \"kafka-inputtopic-0-high-watermark\",\n" +
-			"      \"count\": {\n" +
-			"        \"value\": 1000\n" +
-			"      }\n" +
-			"    }\n" +
-			"  }, \n" +
-			"  \"org.apache.samza.checkpoint.OffsetManagerMetrics\": {\n" +
-			"    \"kafka-inputtopic-0-checkpointed-offset\": {\n" +
-			"      \"name\": \"kafka-inputtopic-0-checkpointed-offset\",\n" +
-			"      \"count\": {\n" +
-			"        \"value\": 200\n" +
-			"      }\n" +
-			"    }\n" +
-			"  }\n" +
-			"}";
-
-	public static final String METRIC_EVENT_STREAM2 = "{\n" +
-			"  \"org.apache.samza.system.kafka.KafkaSystemConsumerMetrics\": {\n" +
-			"    \"kafka-inputtopic-0-high-watermark\": {\n" +
-			"      \"name\": \"kafka-inputtopic-0-high-watermark\",\n" +
-			"      \"count\": {\n" +
-			"        \"value\": 1000\n" +
-			"      }\n" +
-			"    }\n" +
-			"  }, \n" +
-			"  \"org.apache.samza.checkpoint.OffsetManagerMetrics\": {\n" +
-			"    \"kafka-inputtopic-0-checkpointed-offset\": {\n" +
-			"      \"name\": \"kafka-inputtopic-0-checkpointed-offset\",\n" +
-			"      \"count\": {\n" +
-			"        \"value\": 1000\n" +
-			"      }\n" +
-			"    }\n" +
-			"  }\n" +
-			"}";
-
-
-	public static Map<String, ConcurrentHashMap<String, Metric>> getMetricMap(String message) {
-		Type type = new TypeToken<Map<String, ConcurrentHashMap<String, Counter>>>() {}.getType();
-		return (Map<String, ConcurrentHashMap<String, Metric>>) new Gson().fromJson(message, type);
-	}
-
-
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java
deleted file mode 100644
index 0237ae17a4..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/Element.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.ekstep.ep.samza.schema;
-
-public class Element {
-    private String id;
-    private Object value;
-
-    public Element(String id, Object value) {
-        this.id = id;
-        this.value = value;
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public void setId(String id) {
-        this.id = id;
-    }
-
-    public Object getValue() {
-        return value;
-    }
-
-    public void setValue(Object value) {
-        this.value = value;
-    }
-
-    @Override
-    public int hashCode() {
-        return id.hashCode();
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        Element other = (Element) obj;
-        return id.equalsIgnoreCase(other.id) && value.equals(other.value);
-    }
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java
deleted file mode 100644
index d25b2e0fe7..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/schema/MetricEvent.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.ekstep.ep.samza.schema;
-
-import org.joda.time.DateTime;
-
-import java.util.List;
-
-public class MetricEvent {
-
-    private String system;
-    private String subsystem;
-    private Long metricts;
-    private List<Element> metrics;
-    private List<Element> dimensions;
-
-    public String getSystem() {
-        return system;
-    }
-
-    public void setSystem(String system) {
-        this.system = system;
-    }
-
-    public String getSubsystem() {
-        return subsystem;
-    }
-
-    public void setSubsystem(String subsystem) {
-        this.subsystem = subsystem;
-    }
-
-    public Long getMetricts() {
-        return metricts;
-    }
-
-    public void setMetricts(Long metricts) {
-        this.metricts = metricts;
-    }
-
-    public List<Element> getMetrics() {
-        return metrics;
-    }
-
-    public void setMetrics(List<Element> metrics) {
-        this.metrics = metrics;
-    }
-
-    public List<Element> getDimensions() {
-        return dimensions;
-    }
-
-    public void setDimensions(List<Element> dimensions) {
-        this.dimensions = dimensions;
-    }
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java
deleted file mode 100644
index ed2b716702..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/BaseUpdaterTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.exceptions.DriverException;
-import com.datastax.driver.core.querybuilder.Clause;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.core.BaseCacheUpdaterService;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-import redis.clients.jedis.Jedis;
-
-import java.lang.reflect.Type;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.*;
-
-
-public class BaseUpdaterTest {
-
-    private RedisConnect redisConnectMock;
-    private CassandraConnect cassandraConnectMock;
-    private BaseCacheUpdaterService baseUpdater;
-    private Jedis jedisMock = new MockJedis("test");
-    private static int storeId = 5;
-    Gson gson = new Gson();
-    Type type = new TypeToken<Map<String, Object>>() {
-    }.getType();
-
-    @Before
-    public void setUp() {
-        redisConnectMock = mock(RedisConnect.class);
-        cassandraConnectMock = mock(CassandraConnect.class);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        new BaseCacheUpdaterService(redisConnectMock);
-        baseUpdater = new BaseCacheUpdaterService(redisConnectMock, cassandraConnectMock);
-
-        stub(redisConnectMock.getConnection(anyInt())).toReturn(jedisMock);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-    }
-
-    @Test
-    public void shouldAddToCache() {
-        baseUpdater.addToCache("4569876545678", "{\"role\":\"student\",\"type\":\"User\"}", storeId);
-        String value = jedisMock.get("4569876545678");
-        Map<String, Object> parsedData = gson.fromJson(value, type);
-
-        assertEquals("User", parsedData.get("type"));
-        assertEquals("student", parsedData.get("role"));
-
-    }
-
-    @Test
-    public void shouldReadFromCache() {
-        jedisMock.select(storeId);
-        jedisMock.set("4569876545678", "{\"role\":\"teacher\",\"type\":\"Request\"}");
-        String value = baseUpdater.readFromCache("4569876545678", storeId);
-        Map<String, Object> parsedData = gson.fromJson(value, type);
-
-        assertEquals("Request", parsedData.get("type"));
-        assertEquals("teacher", parsedData.get("role"));
-    }
-
-    @Test(expected = DriverException.class)
-    public void shouldHandleCassandraException() throws Exception {
-        Clause userDataClause = QueryBuilder.eq("id", "id");
-
-        when(cassandraConnectMock.find(anyString())).thenThrow(new DriverException("Cassandra Exception"));
-        baseUpdater.readFromCassandra("sunbird", "user", userDataClause);
-    }
-
-    @Test(expected = DriverException.class)
-    public void shouldHandleCassandraExceptionForLocationQuery() throws Exception {
-        List<String> locationIds = new ArrayList<>();
-        locationIds.add("1f56a8458d78df90");
-        Clause locationDataClause = QueryBuilder.in("id", locationIds);
-
-        when(cassandraConnectMock.find(anyString())).thenThrow(new DriverException("Cassandra Exception"));
-        baseUpdater.readFromCassandra("sunbird", "location", locationDataClause);
-    }
-
-    @Test
-    public void shouldHandleNullOrEmptyClause() {
-        List<Row> emptyClauseValue = baseUpdater.readFromCassandra("sunbird", "user", QueryBuilder.eq("", ""));
-        assertEquals(true, emptyClauseValue.isEmpty());
-    }
-
-    @Test
-    public void shouldGetLocationDetailsFromDB() {
-        List<String> locationIds = new ArrayList<>();
-        locationIds.add("1f56a8458d78df90");
-        ArrayList location = new ArrayList(3);
-        location.add(0, "state-name");
-        location.add(1, "district-name");
-        Clause locationDataClause = QueryBuilder.in("id", locationIds);
-        stub(baseUpdater.readFromCassandra("sunbird", "location", locationDataClause)).toReturn(location);
-
-        List<Row> row = baseUpdater.readFromCassandra("sunbird", "location", locationDataClause);
-        assertEquals("state-name", row.get(0));
-        assertEquals("district-name", row.get(1));
-    }
-
-    @Test
-    public void shouldUpdateToCassandra() {
-        Map<String, String> map = new HashMap<>();
-        map.put("device_id", "5975394fjh9543978987593");
-        ArrayList location = new ArrayList(3);
-        location.add(0, "device_id");
-        baseUpdater.updateToCassandra("sunbird", "location", map);
-        Clause locationDataClause = QueryBuilder.in("id", map);
-        stub(baseUpdater.readFromCassandra("sunbird", "location", locationDataClause)).toReturn(location);
-        List<Row> row = baseUpdater.readFromCassandra("sunbird", "location", locationDataClause);
-        Assert.assertNotNull(row.get(0));
-    }
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java
deleted file mode 100644
index 7557ccd0ad..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/JobMetricsTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.container.SamzaContainerContext;
-import org.apache.samza.container.TaskName;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.Metric;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.metrics.MetricsRegistryMap;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.TaskContext;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.fixtures.MetricsFixture;
-import org.ekstep.ep.samza.schema.Element;
-import org.ekstep.ep.samza.schema.MetricEvent;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ConcurrentHashMap;
-
-import static org.junit.Assert.assertEquals;
-import static junit.framework.TestCase.assertTrue;
-import static org.mockito.Mockito.*;
-
-public class JobMetricsTest {
-
-    private TaskContext contextMock;
-    private JobMetrics jobMetricsMock;
-
-    @Before
-    public void setUp() {
-        contextMock = mock(TaskContext.class);
-        MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
-        Counter counter = mock(Counter.class);
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-    }
-
-    @Test
-    public void shouldReturnConsumerLag() {
-
-        jobMetricsMock = new JobMetrics(contextMock, "test-job");
-        jobMetricsMock.clear();
-        Set<SystemStreamPartition> systemStreamPartitions = new HashSet<>();
-        SystemStreamPartition systemStreamPartition =
-                new SystemStreamPartition("kafka", "inputtopic", new Partition(0));
-        systemStreamPartitions.add(systemStreamPartition);
-
-        Map<String, ConcurrentHashMap<String, Metric>> concurrentHashMap =
-                MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1);
-        when(contextMock.getSystemStreamPartitions()).thenReturn(systemStreamPartitions);
-        long consumer_lag = jobMetricsMock.consumerLag(concurrentHashMap);
-        Assert.assertEquals(800, consumer_lag);
-
-    }
-
-    @Test
-    public void shouldReturnZeroConsumerLagWhenAllMessagesAreProcessed() {
-
-        jobMetricsMock = new JobMetrics(contextMock, null);
-
-        Set<SystemStreamPartition> systemStreamPartitions = new HashSet<>();
-        SystemStreamPartition systemStreamPartition =
-                new SystemStreamPartition("kafka", "inputtopic", new Partition(0));
-        systemStreamPartitions.add(systemStreamPartition);
-
-        Map<String, ConcurrentHashMap<String, Metric>> concurrentHashMap =
-                MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM2);
-        when(contextMock.getSystemStreamPartitions()).thenReturn(systemStreamPartitions);
-        long consumer_lag = jobMetricsMock.consumerLag(concurrentHashMap);
-        Assert.assertEquals(0, consumer_lag);
-    }
-
-    /*
-    @Test
-    public void shouldCollectTheMetrics() {
-        Map<String, ConcurrentHashMap<String, Metric>> concurrentHashMap =
-                MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1);
-        SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0));
-        Set<SystemStreamPartition> partitionsSet = new HashSet<>();
-        partitionsSet.add(systemStreamPartition);
-        MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("id");
-        metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics", concurrentHashMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics"));
-        metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics", concurrentHashMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics"));
-
-        TaskName taskName = new TaskName("task");
-        ArrayList<TaskName> list = new ArrayList<TaskName>();
-        list.add(taskName);
-        SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id", mock(Config.class), list, metricsRegistryMap);
-        when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext);
-        when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet);
-        when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry);
-
-        jobMetricsMock = new JobMetrics(contextMock, "test-job");
-        jobMetricsMock.incSuccessCounter();
-        jobMetricsMock.deviceDBUpdateSuccess();
-        jobMetricsMock.deviceCacheUpdateSuccess();
-        jobMetricsMock.incFailedCounter();
-        jobMetricsMock.incSkippedCounter();
-        jobMetricsMock.incErrorCounter();
-        jobMetricsMock.incBatchSuccessCounter();
-        jobMetricsMock.incBatchErrorCounter();
-        jobMetricsMock.incPrimaryRouteSuccessCounter();
-        jobMetricsMock.incSecondaryRouteSuccessCounter();
-        jobMetricsMock.incDuplicateCounter();
-        jobMetricsMock.incCacheHitCounter();
-        jobMetricsMock.incCacheMissCounter();
-        jobMetricsMock.incCacheErrorCounter();
-        jobMetricsMock.incEmptyCacheValueCounter();
-        jobMetricsMock.incProcessedMessageCount();
-        jobMetricsMock.incUnprocessedMessageCount();
-        jobMetricsMock.incDBHitCount();
-        jobMetricsMock.incUserCacheHitCount();
-        jobMetricsMock.incExpiredEventCount();
-        jobMetricsMock.incUserDeclaredHitCount();
-        jobMetricsMock.incIpLocationHitCount();
-        jobMetricsMock.incDialCodesCount();
-        jobMetricsMock.incDialCodesFromApiCount();
-        jobMetricsMock.incDialCodesFromCacheCount();
-
-        String metricsJson = jobMetricsMock.collect();
-        System.out.println(metricsJson);
-        Gson gson = new Gson();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> metrics = (Map<String, Object>) gson.fromJson(metricsJson, Map.class);
-        assertEquals("test-job", metrics.get("job-name"));
-        assertEquals(1.0, metrics.get("cache-hit-count"));
-        assertEquals(1.0, metrics.get("db-hit-count"));
-        assertEquals(1.0, metrics.get("batch-error-count"));
-        assertEquals(1.0, metrics.get("success-message-count"));
-        assertEquals(1.0, metrics.get("cache-empty-values-count"));
-        assertEquals(1.0, metrics.get("failed-message-count"));
-        assertEquals(1.0, metrics.get("unprocessed-message-count"));
-        assertEquals(1.0, metrics.get("expired-event-count"));
-        assertEquals(1.0, metrics.get("duplicate-event-count"));
-        assertEquals(1.0, metrics.get("processed-message-count"));
-        assertEquals(1.0, metrics.get("primary-route-success-count"));
-        assertEquals(1.0, metrics.get("batch-success-count"));
-        assertEquals(1.0, metrics.get("secondary-route-success-count"));
-        assertEquals(1.0, metrics.get("device-cache-update-count"));
-        assertEquals(1.0, metrics.get("cache-error-count"));
-        assertEquals(1.0, metrics.get("error-message-count"));
-        assertEquals(1.0, metrics.get("dialcodes-count"));
-        assertEquals(1.0, metrics.get("dialcodes-api-hit"));
-        assertEquals(1.0, metrics.get("dialcodes-cache-hit"));
-        assertEquals(0.0, metrics.get("partition"));
-        assertEquals(800.0, metrics.get("consumer-lag"));
-    }
-    */
-
-
-    @Test
-    public void validateCollectedMetrics() {
-        Map<String, ConcurrentHashMap<String, Metric>> metricsMap =
-                MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1);
-        MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("id");
-        SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0));
-        Set<SystemStreamPartition> partitionsSet = new HashSet<>();
-        partitionsSet.add(systemStreamPartition);
-        metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics",
-                metricsMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics"));
-        metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics",
-                metricsMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics"));
-        TaskName taskName = new TaskName("task");
-        ArrayList<TaskName> taskList = new ArrayList<>();
-        taskList.add(taskName);
-        SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id", mock(Config.class), taskList, metricsRegistryMap);
-        when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext);
-        when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet);
-        when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry);
-
-        jobMetricsMock = new JobMetrics(contextMock, "test-job");
-        List<String> jobMetricList = new ArrayList<>(Arrays.asList("success-message-count","skipped-message-count","error-message-count",
-                "batch-success-count","batch-error-count","duplicate-event-count"));
-        jobMetricsMock.incSuccessCounter();
-        jobMetricsMock.incSkippedCounter();
-        jobMetricsMock.incErrorCounter();
-        jobMetricsMock.incBatchSuccessCounter();
-        jobMetricsMock.incBatchErrorCounter();
-        jobMetricsMock.incDuplicateCounter();
-        String metricsOutput = jobMetricsMock.collect(jobMetricList);
-        System.out.println(metricsOutput);
-        Gson gson = new Gson();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> metrics = (Map<String, Object>) gson.fromJson(metricsOutput, Map.class);
-        assertEquals("test-job", metrics.get("job-name"));
-        assertEquals(1.0, metrics.get("success-message-count"));
-        assertEquals(1.0, metrics.get("skipped-message-count"));
-        assertEquals(1.0, metrics.get("error-message-count"));
-        assertEquals(1.0, metrics.get("batch-success-count"));
-        assertEquals(1.0, metrics.get("batch-error-count"));
-        assertEquals(1.0, metrics.get("duplicate-event-count"));
-        assertEquals(0.0, metrics.get("partition"));
-        assertEquals(800.0, metrics.get("consumer-lag"));
-        assertTrue(metrics.containsKey("metricts") && metrics.get("metricts") != null);
-    }
-
-    @Test
-    public void generatePrometheusMetricStructure() {
-        Map<String, ConcurrentHashMap<String, Metric>> concurrentHashMap =
-                MetricsFixture.getMetricMap(MetricsFixture.METRIC_EVENT_STREAM1);
-
-        MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap("metric-registry");
-        metricsRegistryMap.metrics().put("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics",
-                concurrentHashMap.get("org.apache.samza.system.kafka.KafkaSystemConsumerMetrics"));
-        metricsRegistryMap.metrics().put("org.apache.samza.checkpoint.OffsetManagerMetrics",
-                concurrentHashMap.get("org.apache.samza.checkpoint.OffsetManagerMetrics"));
-
-        SystemStreamPartition systemStreamPartition = new SystemStreamPartition("kafka", "inputtopic", new Partition(0));
-        Set<SystemStreamPartition> partitionsSet = new HashSet<>();
-        partitionsSet.add(systemStreamPartition);
-
-        TaskName taskName = new TaskName("generate-prometheus-metrics-task");
-        ArrayList<TaskName> taskList = new ArrayList<>(Collections.singletonList(taskName));
-        SamzaContainerContext samzaContainerContext = new SamzaContainerContext("id",
-                mock(Config.class), taskList, metricsRegistryMap);
-        when(contextMock.getSamzaContainerContext()).thenReturn(samzaContainerContext);
-        when(contextMock.getSystemStreamPartitions()).thenReturn(partitionsSet);
-        when(contextMock.getMetricsRegistry()).thenReturn(samzaContainerContext.metricsRegistry);
-
-        jobMetricsMock = new JobMetrics(contextMock, "test-job");
-
-        List<String> jobMetricList = new ArrayList<>(Arrays.asList("success-message-count","skipped-message-count","error-message-count",
-                "batch-success-count","batch-error-count","duplicate-event-count"));
-        jobMetricsMock.incSuccessCounter();
-        jobMetricsMock.incSkippedCounter();
-        jobMetricsMock.incErrorCounter();
-        jobMetricsMock.incBatchSuccessCounter();
-        jobMetricsMock.incBatchErrorCounter();
-        jobMetricsMock.incDuplicateCounter();
-        String metricsOutput = jobMetricsMock.generateMetrics(jobMetricList);
-
-        SimpleDateFormat dt = new SimpleDateFormat("yyyy-MM-dd");
-        Gson gson = new Gson();
-        @SuppressWarnings("unchecked")
-        MetricEvent metrics = gson.fromJson(metricsOutput, MetricEvent.class);
-        assertEquals("samza", metrics.getSystem());
-        assertEquals("pipeline-metrics", metrics.getSubsystem());
-        assertEquals(dt.format(new Date()), dt.format(new Date(metrics.getMetricts())));
-        assertEquals(7, metrics.getMetrics().size());
-        assertEquals(2, metrics.getDimensions().size());
-
-
-        List<Element> metricList = metrics.getMetrics();
-        assertTrue(metricList.contains(new Element("success-message-count", 1.0)));
-        assertTrue(metricList.contains(new Element("skipped-message-count", 1.0)));
-        assertTrue(metricList.contains(new Element("error-message-count", 1.0)));
-        assertTrue(metricList.contains(new Element("batch-success-count", 1.0)));
-        assertTrue(metricList.contains(new Element("batch-error-count", 1.0)));
-        assertTrue(metricList.contains(new Element("duplicate-event-count", 1.0)));
-        assertTrue(metricList.contains(new Element("consumer-lag", 800.0)));
-
-        List<Element> dimsList = metrics.getDimensions();
-        assertTrue(dimsList.contains(new Element("job-name", "test-job")));
-        assertTrue(dimsList.contains(new Element("partition", 0.0)));
-    }
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java
deleted file mode 100644
index e627c8fa8f..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/PostgresConnectTest.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.opentable.db.postgres.embedded.EmbeddedPostgres;
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.util.PostgresConnect;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.sql.*;
-
-import static org.junit.Assert.*;
-
-import static org.mockito.Mockito.*;
-
-public class PostgresConnectTest {
-
-    private Config configMock;
-    private PostgresConnect postgresConnect;
-    private Connection connection;
-    private Connection resetConnection;
-
-    @Before
-    public void setUp() throws Exception {
-        configMock = mock(Config.class);
-
-        EmbeddedPostgres.builder().setPort(5430).start();
-
-        stub(configMock.get("postgres.user")).toReturn("postgres");
-        stub(configMock.get("postgres.password")).toReturn("postgres");
-        stub(configMock.get("postgres.db")).toReturn("postgres");
-        stub(configMock.get("postgres.host","127.0.0.1")).toReturn("localhost");
-        stub(configMock.getInt("postgres.port", 5432)).toReturn(5430);
-
-        postgresConnect = new PostgresConnect(configMock);
-    }
-
-    @Test
-    public void shouldVerifyPostgresConnect() throws Exception {
-        connection = postgresConnect.getConnection();
-        assertNotNull(connection);
-
-        postgresConnect.execute("CREATE TABLE device_table(id text PRIMARY KEY, channel text);");
-        postgresConnect.execute("INSERT INTO device_table(id,channel)  VALUES('12345','custchannel');");
-
-        Statement st = connection.createStatement();
-        ResultSet rs = st.executeQuery("SELECT * FROM device_table where id='12345';");
-        while(rs.next()) {
-            assertEquals("12345", rs.getString("id"));
-            assertEquals("custchannel", rs.getString("channel"));
-        }
-
-        resetConnection = postgresConnect.resetConnection();
-        assertNotNull(resetConnection);
-
-        postgresConnect.closeConnection();
-    }
-
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java
deleted file mode 100644
index 2604049037..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/RestUtilTest.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import kong.unirest.UnirestException;
-import okhttp3.mockwebserver.MockResponse;
-import okhttp3.mockwebserver.MockWebServer;
-import org.ekstep.ep.samza.util.RestUtil;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertNotNull;
-
-public class RestUtilTest {
-    @Test
-    public void shouldInvokeHTTPCall() throws IOException {
-        MockWebServer server = new MockWebServer();
-        server.enqueue(new MockResponse().setBody("hello, world!"));
-        server.enqueue(new MockResponse().setHeader("Authorization", ""));
-        try {
-            server.start(3000);
-        } catch (IOException e) {
-            System.out.println("Exception" + e);
-        }
-        server.url("http://127.0.0.1:3000/api/dialcode/v3/read/");
-        Map<String, String> headers = new HashMap<>();
-        headers.put("Authorization", "");
-        try {
-            String response = new RestUtil().get("http://127.0.0.1:3000/api/dialcode/v3/read/", headers);
-            // TODO: Assertion are failing need to add
-            System.out.println("response is" + response);
-            assertNotNull(response);
-        } catch (UnirestException e) {
-            System.out.println("Exception is" + e);
-        } finally {
-            server.shutdown();
-        }
-
-
-    }
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java
deleted file mode 100644
index 77e8a28820..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/DeDupEngineTest.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.ekstep.ep.samza.task.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import redis.clients.jedis.Jedis;
-import redis.embedded.RedisServer;
-
-public class DeDupEngineTest {
-	
-	private RedisServer redisServer;
-	private Config configMock;
-	private RedisConnect redisConnect;
-	
-	@Before
-    public void setUp() throws IOException {
-		redisServer = new RedisServer(6379);
-		redisServer.start();
-		configMock = Mockito.mock(Config.class);
-		when(configMock.get("redis.host", "localhost")).thenReturn("localhost");
-		when(configMock.getInt("redis.port", 6379)).thenReturn(6379);
-		when(configMock.getInt("redis.connection.max", 2)).thenReturn(2);
-		when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2);
-		when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1);
-		when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120);
-		when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300);
-		redisConnect = new RedisConnect(configMock);
-	}
-	
-	@After
-    public void tearDown() {
-		redisServer.stop();
-	}
-	
-	@Test
-    public void shouldVerifyDedupEngine() {
-		
-		DeDupEngine engine = new DeDupEngine(redisConnect, 1, 100);
-		
-		engine.storeChecksum("1234");
-		
-		assertTrue(engine.isUniqueEvent("2345"));
-		assertTrue(!engine.isUniqueEvent("1234"));
-		
-		Jedis jedis = engine.getRedisConnection();
-		assertNotNull(jedis);
-		assertEquals(Long.valueOf(1), jedis.getDB());
-		
-		redisServer.stop();
-		try {
-			engine.storeChecksum("1234");
-		} catch(Exception ex) {
-			assertNotNull(ex);
-		}
-		
-		try {
-			assertTrue(engine.isUniqueEvent("2345"));
-		} catch(Exception ex) {
-			assertNotNull(ex);
-		} finally {
-			redisServer.start();
-			engine.storeChecksum("1234");
-			assertTrue(engine.isUniqueEvent("2345"));
-			assertTrue(!engine.isUniqueEvent("1234"));
-		}
-		
-	}
-
-}
diff --git a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java b/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java
deleted file mode 100644
index 93e2870778..0000000000
--- a/data-pipeline/ep-core/src/test/java/org/ekstep/ep/samza/task/util/RedisConnectTest.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package org.ekstep.ep.samza.task.util;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-
-import redis.clients.jedis.Jedis;
-import redis.embedded.RedisServer;
-
-public class RedisConnectTest {
-	
-	private RedisServer redisServer;
-	private Config configMock;
-	private RedisConnect redisConnect;
-	
-	@Before
-    public void setUp() throws IOException {
-		redisServer = new RedisServer(6379);
-		redisServer.start();
-		configMock = Mockito.mock(Config.class);
-		when(configMock.get("redis.host", "localhost")).thenReturn("localhost");
-		when(configMock.getInt("redis.port", 6379)).thenReturn(6379);
-		when(configMock.getInt("redis.connection.max", 2)).thenReturn(2);
-		when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2);
-		when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1);
-		when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120);
-		when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300);
-		redisConnect = new RedisConnect(configMock);
-	}
-	
-	@After
-    public void tearDown() {
-		redisServer.stop();
-	}
-	
-	@Test
-    public void shouldVerifyRedisConnect() {
-		Jedis jedis = redisConnect.getConnection(0);
-		assertEquals(Long.valueOf(0), jedis.getDB());
-		assertNotNull(jedis);
-		jedis.close();
-		
-		jedis = redisConnect.getConnection(1, 0);
-		assertNotNull(jedis);
-		assertEquals(Long.valueOf(1), jedis.getDB());
-		jedis.set("key", "value");
-		assertEquals("value", jedis.get("key"));
-		jedis.close();
-		
-	}
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/pom.xml b/data-pipeline/ep-telemetry-reader/pom.xml
deleted file mode 100644
index c46811475e..0000000000
--- a/data-pipeline/ep-telemetry-reader/pom.xml
+++ /dev/null
@@ -1,90 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>jobs</artifactId>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <version>0.0.1</version>
-    </parent>
-    <modelVersion>4.0.0</modelVersion>
-
-    <artifactId>ep-telemetry-reader</artifactId>
-    <version>0.0.3</version>
-    <packaging>jar</packaging>
-    <name>ep-telemetry-reader</name>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.6.2</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
-
-</project>
\ No newline at end of file
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java
deleted file mode 100644
index bff396e749..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Events.java
+++ /dev/null
@@ -1,141 +0,0 @@
-package org.ekstep.ep.samza.events.domain;
-
-
-import com.google.gson.Gson;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.reader.Telemetry;
-
-import java.util.List;
-import java.util.Map;
-
-public abstract class Events {
-
-    protected final Telemetry telemetry;
-
-    protected Path path;
-
-    public Events(Map<String, Object> map) {
-        this.telemetry = new Telemetry(map);
-        path = new Path();
-    }
-
-    public String getChecksum() {
-        String checksum = id();
-        if (checksum != null)
-            return checksum;
-        return mid();
-    }
-
-    public String id() {
-        NullableValue<String> checksum = telemetry.read("metadata.checksum");
-        return checksum.value();
-    }
-
-    public Map<String, Object> getMap() {
-        return telemetry.getMap();
-    }
-
-    public String getJson() {
-        Gson gson = new Gson();
-        String json = gson.toJson(getMap());
-        return json;
-    }
-
-    public String mid() {
-        NullableValue<String> checksum = telemetry.read("mid");
-        return checksum.value();
-    }
-
-    public String did() {
-        NullableValue<String> did = telemetry.read("dimensions.did");
-        return did.isNull() ? telemetry.<String>read("context.did").value() : did.value();
-    }
-
-    public String eid() {
-        NullableValue<String> eid = telemetry.read("eid");
-        return eid.value();
-    }
-
-    @Override
-    public String toString() {
-        return "Event{" + "telemetry=" + telemetry + '}';
-    }
-
-
-    public void updateTs(String value) {
-        telemetry.add("@timestamp", value);
-    }
-
-    public String pid() {
-        NullableValue<String> pid = telemetry.read("context.pdata.pid");
-        return pid.value();
-    }
-
-    public String version() {
-        return (String) telemetry.read("ver").value();
-    }
-
-    public String producerId() {
-        NullableValue<String> producerId = telemetry.read("context.pdata.id");
-        return producerId.value();
-    }
-
-    public final String producerPid() {
-        NullableValue<String> producerPid = telemetry.read("context.pdata.pid");
-        return producerPid.value();
-    }
-
-
-    public Long ets() {
-        NullableValue<Object> ets = telemetry.read("ets");
-        if (ets.value().getClass().equals(Double.class)) {
-            return ((Double) ets.value()).longValue();
-        }
-        return ((Long) ets.value());
-    }
-
-    public String channel() {
-        NullableValue<String> channel = telemetry.read("dimensions.channel");
-        return channel.isNull() ? telemetry.<String>read("context.channel").value() : channel.value();
-    }
-
-    public String actorId() {
-        NullableValue<String> actorid = telemetry.read("uid");
-        return actorid.isNull() ? telemetry.<String>read("actor.id").value() : actorid.value();
-    }
-
-
-    public String actorType() {
-        NullableValue<String> actortype = telemetry.read("actor.type");
-        return actortype.value();
-    }
-
-    public String objectID() {
-        if (objectFieldsPresent()) {
-            return telemetry.<String>read("object.id").value();
-        } else return null;
-    }
-
-    public String objectType() {
-        if (objectFieldsPresent()) {
-            return telemetry.<String>read("object.type").value();
-        } else return null;
-    }
-
-    public boolean objectFieldsPresent() {
-        String objectId = telemetry.<String>read("object.id").value();
-        String objectType = telemetry.<String>read("object.type").value();
-        return null != objectId && null != objectType && !objectId.isEmpty() && !objectType.isEmpty();
-    }
-
-    public String edataType() {
-        return telemetry.<String>read("edata.type").value();
-    }
-
-    public List<Map<String, Object>> edataItems() {
-        return telemetry.<List<Map<String, Object>>>read("edata.items").value();
-    }
-
-
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java
deleted file mode 100644
index b12ef5fb94..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/events/domain/Path.java
+++ /dev/null
@@ -1,94 +0,0 @@
-package org.ekstep.ep.samza.events.domain;
-
-public class Path {
-
-    public String metadata() {
-        return "metadata";
-    }
-
-    public String flags() {
-        return "flags";
-    }
-
-    public String dimensionsDid() {
-        return "dimensions.did";
-    }
-
-    public String contextDid() {
-        return "context.did";
-    }
-
-    public String ets() {
-        return "ets";
-    }
-
-    public String ts() {
-        return "ts";
-    }
-
-    public String mid() {
-        return "mid";
-    }
-
-    public String eid() {
-        return "eid";
-    }
-
-    public String deviceData() {
-        return "devicedata";
-    }
-
-    public String loc() {
-        return "edata.loc";
-    }
-
-    public String userData() {
-        return "userdata";
-    }
-
-    public String contentData() {
-        return "contentdata";
-    }
-
-    public String dialCodeData() {
-        return "dialcodedata";
-    }
-
-    public String collectionData() {
-        return "collectiondata";
-    }
-
-    public String ver() {
-        return "ver";
-    }
-
-    public String edata() {
-        return "edata";
-    }
-
-    public String checksum() {
-        return "metadata.checksum";
-    }
-
-    public String channel() {
-        return "channel";
-    }
-
-    public String derivedLocationData() {
-        return "derivedlocationdata";
-    }
-
-    public String stateKey() {
-        return "state";
-    }
-
-    public String districtKey() {
-        return "district";
-    }
-
-    public String locDerivedFromKey() {
-        return "from";
-    }
-
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java
deleted file mode 100644
index 2b7d612957..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullParent.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-import java.text.MessageFormat;
-
-import org.ekstep.ep.samza.core.Logger;
-
-class NullParent implements ParentType {
-    static Logger LOGGER = new Logger(NullParent.class);
-    Object parent;
-    String childKey;
-
-    NullParent(Object parent, String childKey) {
-        this.parent = parent;
-        this.childKey = childKey;
-    }
-
-    @Override
-    public <T> T readChild() {
-        LOGGER.warn(null, MessageFormat.format("NULL PARENT READ CHILD INVOKED FOR PARENT: {0}, CHILD KEY: {1}", parent, childKey));
-        return null;
-    }
-
-    @Override
-    public void addChild(Object value) {
-        LOGGER.warn(null, MessageFormat.format("NULL PARENT ADD CHILD INVOKED FOR PARENT: {0}, CHILD KEY: {1}", parent, childKey));
-    }
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java
deleted file mode 100644
index 679e9bf241..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/NullableValue.java
+++ /dev/null
@@ -1,46 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-public class NullableValue<T> {
-    private T value;
-
-    public NullableValue(T value) {
-        this.value = value;
-    }
-
-    public T value() {
-        return value;
-    }
-
-    public boolean isNull() {
-        return value == null;
-    }
-
-    public T valueOrDefault(T defaultValue) {
-        if (isNull())
-            return defaultValue;
-
-        return value();
-    }
-
-    @Override
-    public String toString() {
-        return "NullableValue{" +
-                "value=" + value +
-                '}';
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        NullableValue<?> that = (NullableValue<?>) o;
-
-        return value != null ? value.equals(that.value) : that.value == null;
-    }
-
-    @Override
-    public int hashCode() {
-        return value != null ? value.hashCode() : 0;
-    }
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java
deleted file mode 100644
index 85cccd36c6..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentListOfMap.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-import sun.reflect.generics.reflectiveObjects.NotImplementedException;
-
-import java.util.List;
-import java.util.Map;
-
-import org.ekstep.ep.samza.core.Logger;
-
-class ParentListOfMap implements ParentType {
-    //TODO#: Make this class more genic.
-    static Logger LOGGER = new Logger(ParentListOfMap.class);
-    List<Map<String, Object>> list;
-    String childKey;
-
-    ParentListOfMap(List<Map<String, Object>> list, String childKey) {
-        this.list = list;
-        this.childKey = childKey;
-    }
-
-    @Override
-    public <T> T readChild() {
-        if (list == null) {
-            return null;
-        }
-        for (Object itemsObject : list) {
-            if (!(itemsObject instanceof Map)) {
-                continue;
-            }
-            Map<String, Object> items = (Map<String, Object>) itemsObject;
-            if (items.containsKey(childKey)) {
-                Object o = items.get(childKey);
-                if (o instanceof List && ((List) o).size() > 0) {
-                    return (T) ((List) o).get(0);
-                }
-            }
-        }
-        return null;
-    }
-
-    @Override
-    public void addChild(Object value) {
-        throw new NotImplementedException();
-    }
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java
deleted file mode 100644
index f34bfe6206..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentMap.java
+++ /dev/null
@@ -1,31 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-import java.util.Map;
-
-import org.ekstep.ep.samza.core.Logger;
-
-class ParentMap implements ParentType {
-    static Logger LOGGER = new Logger(ParentMap.class);
-    Map<String, Object> map;
-    String childKey;
-
-    ParentMap(Map<String, Object> map, String childKey) {
-        this.map = map;
-        this.childKey = childKey;
-    }
-
-    @Override
-    public <T> T readChild() {
-        if (map != null && map.containsKey(childKey) && map.get(childKey) != null) {
-            Object child = map.get(childKey);
-            return (T) child;
-        }
-        return null;
-    }
-
-    @Override
-    public void addChild(Object value) {
-        if (map != null)
-            map.put(childKey, value);
-    }
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java
deleted file mode 100644
index 1424b629a7..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/ParentType.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-public interface ParentType {
-    <T> T readChild();
-
-    void addChild(Object value);
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java
deleted file mode 100644
index 3980a66737..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/Telemetry.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-
-import org.ekstep.ep.samza.core.Logger;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-
-import static java.text.MessageFormat.format;
-
-public class Telemetry {
-    private Map<String, Object> map;
-    private Logger logger = new Logger(this.getClass());
-
-    public Telemetry(Map<String, Object> map) {
-        this.map = map;
-    }
-
-    public boolean add(String keyPath, Object value) {
-        try {
-            ParentType lastParent = lastParentMap(map, keyPath);
-            lastParent.addChild(value);
-            return true;
-        } catch (Exception e) {
-            logger.error("", format("Couldn't add value:{0} at  key: {0} for event:{1}", value, keyPath, map), e);
-        }
-        return false;
-    }
-
-    public Map<String, Object> getMap() {
-        return map;
-    }
-
-    public <T> NullableValue<T> read(String keyPath) {
-        try {
-            ParentType parentMap = lastParentMap(map, keyPath);
-            return new NullableValue<T>(parentMap.<T>readChild());
-        } catch (Exception e) {
-            logger.error("", format("Couldn't get key: {0} from event:{1}", keyPath, map), e);
-            return new NullableValue<T>(null);
-        }
-    }
-
-    public <T> NullableValue<T> readOrDefault(String keyPath, T defaultValue) {
-        return read(keyPath).isNull() ? new NullableValue<T>(defaultValue) : read(keyPath);
-    }
-
-    public <T> T mustReadValue(String keyPath) throws TelemetryReaderException {
-        NullableValue<T> val = read(keyPath);
-        if (val.isNull()) {
-            NullableValue<String> eid = read("eid");
-            String message = keyPath + " is not available in the event";
-            if (!eid.isNull()) {
-                message = keyPath + " is not available in " + eid.value();
-            }
-            throw new TelemetryReaderException(message);
-        }
-
-        return val.value();
-    }
-
-    private ParentType lastParentMap(Map<String, Object> map, String keyPath) {
-        Object parent = map;
-        String[] keys = keyPath.split("\\.");
-        int lastIndex = keys.length - 1;
-        if (keys.length > 1) {
-            for (int i = 0; i < lastIndex && parent != null; i++) {
-                Object o;
-                if (parent instanceof Map) {
-                    o = new ParentMap((Map<String, Object>) parent, keys[i]).readChild();
-                } else if (parent instanceof List) {
-                    o = new ParentListOfMap((List<Map<String, Object>>) parent, keys[i]).readChild();
-                } else {
-                    o = new NullParent(parent, keys[i]).readChild();
-                }
-                parent = o;
-            }
-        }
-        String lastKeyInPath = keys[lastIndex];
-        if (parent instanceof Map) {
-            return new ParentMap((Map<String, Object>) parent, lastKeyInPath);
-        } else if (parent instanceof List) {
-            return new ParentListOfMap((List<Map<String, Object>>) parent, lastKeyInPath);
-        } else {
-            return new NullParent(parent, lastKeyInPath);
-        }
-    }
-
-    @Override
-    public String toString() {
-        return "Telemetry{" +
-                "map=" + map +
-                '}';
-    }
-
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) return true;
-        if (o == null || getClass() != o.getClass()) return false;
-
-        Telemetry telemetry = (Telemetry) o;
-
-        return map != null ? map.equals(telemetry.map) : telemetry.map == null;
-    }
-
-    @Override
-    public int hashCode() {
-        return map != null ? map.hashCode() : 0;
-    }
-
-    public String id() {
-        return this.<String>read("metadata.checksum").value();
-    }
-
-    public <T> void addFieldIfAbsent(String fieldName, T value) {
-        if (read(fieldName).isNull()) {
-            add(fieldName, value);
-        }
-    }
-
-    public long getEts() throws TelemetryReaderException {
-        double ets = this.<Double>mustReadValue("ets");
-        return (long) ets;
-    }
-
-    public String getAtTimestamp(){
-        Object timestamp = this.read("@timestamp").value();
-        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
-        if ( timestamp instanceof Number){
-            Date date = new Date(((Number) timestamp).longValue());
-            return simpleDateFormat.format(date);
-        } else {
-            return this.<String>read("@timestamp").value();
-        }
-    }
-    
-    public String getSyncts(){
-        Object timestamp = this.read("syncts").value();
-        SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
-        if ( timestamp instanceof Number){
-            Date date = new Date(((Number) timestamp).longValue());
-            return simpleDateFormat.format(date);
-        } else {
-            return this.<String>read("syncts").value();
-        }
-    }
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java b/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java
deleted file mode 100644
index 527c1e6e02..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/main/java/org/ekstep/ep/samza/reader/TelemetryReaderException.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-public class TelemetryReaderException extends Exception {
-    public TelemetryReaderException(String message) {
-        super(message);
-    }
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java
deleted file mode 100644
index 8bacf6df7e..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventFixture.java
+++ /dev/null
@@ -1,102 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.util.Map;
-
-public class EventFixture {
-
-    public static final String IMPRESSION_EVENT_MISSING_FIELDS = "{\n" +
-            "  \"eid\": \"IMPRESSION\",\n" +
-            "  \"ets\": 1574945199426,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"IMPRESSION:0093c96434557b2ead169c7156e95770\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" +
-            "    \"type\": \"User\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.portal\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"pid\": \"sunbird-portal\"\n" +
-            "    },\n" +
-            "    \"env\": \"public\",\n" +
-            "    \"sid\": \"1f077b10-11dd-11ea-bbdc-fbe7180a137e\",\n" +
-            "    \"did\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    },\n" +
-            "    \"uid\": \"anonymous\"\n" +
-            "  },\n" +
-            "  \"object\": {},\n" +
-            "  \"tags\": [\n" +
-            "    \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "  ],\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"view\",\n" +
-            "    \"pageid\": \"landing\",\n" +
-            "    \"subtype\": \"init\",\n" +
-            "    \"uri\": \"https://diksha.gov.in/\",\n" +
-            "    \"visits\": []\n" +
-            "  }\n" +
-            "}";
-
-    public static final String IMPRESSION_EVENT = "{\n" +
-            "  \"eid\": \"IMPRESSION\",\n" +
-            "  \"ets\": 1574945199426,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"IMPRESSION:0093c96434557b2ead169c7156e95770\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" +
-            "    \"type\": \"User\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.portal\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"pid\": \"sunbird-portal\"\n" +
-            "    },\n" +
-            "    \"env\": \"public\",\n" +
-            "    \"sid\": \"1f077b10-11dd-11ea-bbdc-fbe7180a137e\",\n" +
-            "    \"did\": \"f1a99b5cf111be23bd0e8d48a50458c6\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    },\n" +
-            "    \"uid\": \"anonymous\"\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"type\": \"content\",\n" +
-            "    \"id\": \"658374_49785\"\n" +
-            "  },\n" +
-            "  \"tags\": [\n" +
-            "    \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "  ],\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"view\",\n" +
-            "    \"pageid\": \"landing\",\n" +
-            "    \"subtype\": \"init\",\n" +
-            "    \"uri\": \"https://diksha.gov.in/\",\n" +
-            "    \"visits\": []\n" +
-            "  },\n" +
-            "  \"metadata\": {\n" +
-            "    \"checksum\": \"55437853\"\n" +
-            "  },\n" +
-            "  \"@timestamp\": 1575265622000,\n" +
-            "  \"syncts\": 56543875684\n" +
-            "}";
-
-
-
-
-
-
-    public static Map<String, Object> getMap(String message) {
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
deleted file mode 100644
index 8e909fb7e9..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/EventTest.java
+++ /dev/null
@@ -1,168 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Map;
-
-public class EventTest {
-
-    @Test
-    public void shouldReturnMid() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("IMPRESSION:0093c96434557b2ead169c7156e95770", event.mid());
-    }
-
-    @Test
-    public void shouldReturnDid() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("f1a99b5cf111be23bd0e8d48a50458c6", event.did());
-    }
-
-    @Test
-    public void shouldReturnEid() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("IMPRESSION", event.eid());
-    }
-
-    @Test
-    public void shouldReturnPid() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("sunbird-portal", event.pid());
-    }
-
-    @Test
-    public void shouldReturnVersion() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("3.0", event.version());
-    }
-
-    @Test
-    public void shouldReturunContextPID() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("sunbird-portal", event.producerPid());
-    }
-
-    @Test
-    public void shouldReturunEts() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Long ets = 1574945199426L;
-        Assert.assertEquals(ets, event.ets());
-    }
-
-    @Test
-    public void shouldReturnChannel() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("505c7c48ac6dc1edc9b08f21db5a571d", event.channel());
-    }
-
-    @Test
-    public void shouldReturnActorId() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("f1a99b5cf111be23bd0e8d48a50458c6", event.actorId());
-    }
-
-    @Test
-    public void shouldReturnActorType() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("User", event.actorType());
-    }
-
-    @Test
-    public void shouldReturnObjectId() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("658374_49785", event.objectID());
-    }
-
-    @Test
-    public void shoudlReturnNullWhenObjectIdIsNotPresent() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS));
-        Assert.assertNull(event.objectID());
-    }
-
-    @Test
-    public void shouldReturnProducerPID() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("prod.diksha.portal", event.producerId());
-    }
-
-    @Test
-    public void shouldReturnObjectType() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("content", event.objectType());
-    }
-
-    @Test
-    public void shouldReturnNullWhenObjectTypeIsNotPresent() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS));
-        Assert.assertNull(event.objectType());
-    }
-
-    @Test
-    public void shouldReturnEdataType() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals("view", event.edataType());
-    }
-
-    @Test
-    public void shouldGetEventJsonObject() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(event.getJson());
-    }
-
-    @Test
-    public void shouldGetTheMetaDataId() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(event.id());
-    }
-
-    @Test
-    public void shouldGetTheCheckSumValue() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(event.getChecksum());
-    }
-
-    @Test
-    public void shouldGetTheStringObject() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(event.toString());
-    }
-
-    @Test
-    public void shouldUpdateTheTSValue() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        event.updateTs("5468376530");
-        Assert.assertEquals(event.getData().value(), "5468376530");
-    }
-
-    @Test
-    public void shouldGetMidValueIfTheCheckSumIsNotPresent() {
-        Event event = new Event(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS));
-
-        Assert.assertEquals(event.getChecksum(), "IMPRESSION:0093c96434557b2ead169c7156e95770");
-    }
-
-
-}
-
-
-/**
- * Creating a dummy event class to to test the base abstract class object
- */
-
-class Event extends Events {
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public <T> NullableValue<T> getData() {
-        return telemetry.read("@timestamp");
-    }
-
-}
-
-
-
diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java
deleted file mode 100644
index 4fbeb5934a..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/domain/PathTest.java
+++ /dev/null
@@ -1,99 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Path;
-import org.junit.Assert;
-import org.junit.Test;
-
-public class PathTest {
-    private Path path = new Path();
-
-    @Test
-    public void shouldGetMetadataName() {
-        Assert.assertEquals("metadata", path.metadata());
-    }
-    @Test
-    public void shouldGetFlagsName() {
-        Assert.assertEquals("flags", path.flags());
-    }
-    @Test
-    public void shouldGetDimensionDid() {
-        Assert.assertEquals("dimensions.did", path.dimensionsDid());
-    }
-    @Test
-    public void shouldGetContextDid() {
-        Assert.assertEquals("context.did", path.contextDid());
-    }
-    @Test
-    public void shouldGetTheEts() {
-        Assert.assertEquals("ets", path.ets());
-    }
-    @Test
-    public void shouldGetCheckSumKey() {
-        Assert.assertEquals("metadata.checksum", path.checksum());
-    }
-    @Test
-    public void shouldGetTimeStampKey() {
-        Assert.assertEquals("ts", path.ts());
-    }
-    @Test
-    public void shouldGetMidKey() {
-        Assert.assertEquals("mid", path.mid());
-    }
-    @Test
-    public void shouldGetEidKey() {
-        Assert.assertEquals("eid", path.eid());
-    }
-    @Test
-    public void shouldGetDeviceDataKey() {
-        Assert.assertEquals("devicedata", path.deviceData());
-    }
-    @Test
-    public void shouldGetLocationKey() {
-        Assert.assertEquals("edata.loc", path.loc());
-    }
-    @Test
-    public void shouldGetUserDataKey() {
-        Assert.assertEquals("userdata", path.userData());
-    }
-    @Test
-    public void shouldGetContentDataKey() {
-        Assert.assertEquals("contentdata", path.contentData());
-    }
-    @Test
-    public void shouldGetDialCodeDataKey() {
-        Assert.assertEquals("dialcodedata", path.dialCodeData());
-    }
-    @Test
-    public void shouldGetCollectionDataKey() {
-        Assert.assertEquals("collectiondata", path.collectionData());
-    }
-    @Test
-    public void shouldGetDerivedLocationDataKey() {
-        Assert.assertEquals("derivedlocationdata", path.derivedLocationData());
-    }
-    @Test
-    public void shouldGetStateKey() {
-        Assert.assertEquals("state", path.stateKey());
-    }
-    @Test
-    public void shouldGetDistrictKey() {
-        Assert.assertEquals("district", path.districtKey());
-    }
-    @Test
-    public void shouldGetLocationDerivedFromKey() {
-        Assert.assertEquals("from", path.locDerivedFromKey());
-    }
-    @Test
-    public void shouldGetEdataKey() {
-        Assert.assertEquals("edata", path.edata());
-    }
-    @Test
-    public void shouldGetChannelKey() {
-        Assert.assertEquals("channel", path.channel());
-    }
-    @Test
-    public void shouldGetVersionKey() {
-        Assert.assertEquals("ver", path.ver());
-    }
-
-}
diff --git a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java b/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java
deleted file mode 100644
index 5f4b225222..0000000000
--- a/data-pipeline/ep-telemetry-reader/src/test/java/org/ekstep/ep/samza/reader/TelemetryTest.java
+++ /dev/null
@@ -1,266 +0,0 @@
-package org.ekstep.ep.samza.reader;
-
-import org.ekstep.ep.samza.domain.EventFixture;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-
-public class TelemetryTest {
-
-
-    @Test
-    public void shouldReadTheDefaultValue() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT_MISSING_FIELDS));
-        Assert.assertEquals(telemetry.readOrDefault("context.channel", "in.sunbird").value(), "in.sunbird");
-    }
-
-    @Test
-    public void shouldReadTheActualValue() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertEquals(telemetry.readOrDefault("context.channel", "in.sunbird").value(), "505c7c48ac6dc1edc9b08f21db5a571d");
-    }
-
-
-    @Test
-    public void shouldGetFirstStringValue() {
-        Map hashMap = new HashMap<String, String>();
-        hashMap.put("key1", "value1");
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1");
-        assertEquals("value1", nullableValue.value());
-        assertFalse(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetFirstMapValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedObject = new HashMap<String, String>();
-        nestedObject.put("key2", "get");
-        hashMap.put("key1", nestedObject);
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1");
-        assertEquals(nestedObject, nullableValue.value());
-        assertFalse(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetOneNestedStringValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedObject = new HashMap<String, String>();
-        nestedObject.put("key2", "get");
-        hashMap.put("key1", nestedObject);
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1.key2");
-        assertEquals("get", nullableValue.value());
-        assertFalse(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetOneNestedMapValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, Object> nestedObject = new HashMap<String, Object>();
-        HashMap<String, String> otherNestedObject = new HashMap<String, String>();
-        otherNestedObject.put("key3", "get");
-        nestedObject.put("key2", otherNestedObject);
-        hashMap.put("key1", nestedObject);
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1.key2");
-        assertEquals(otherNestedObject, nullableValue.value());
-        assertFalse(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetTwoNestedValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, Object> nestedObject = new HashMap<String, Object>();
-        HashMap<String, String> otherNestedObject = new HashMap<String, String>();
-        otherNestedObject.put("key3", "get");
-        nestedObject.put("key2", otherNestedObject);
-        hashMap.put("key1", nestedObject);
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1.key2.key3");
-        assertEquals("get", nullableValue.value());
-        assertFalse(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetNullValueForMissingKey() {
-        Map hashMap = new HashMap<String, Object>();
-        hashMap.put("key1", "get");
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("invalidKey");
-        assertEquals(null, nullableValue.value());
-        assertTrue(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldNotFailWhenReadingWrongNesting() {
-        Map hashMap = new HashMap<String, Object>();
-        hashMap.put("key1", "get");
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("key1.invalidKey");
-        assertEquals(null, nullableValue.value());
-        assertTrue(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldGetNullValueForBothMissingNestedKey() {
-        Map hashMap = new HashMap<String, Object>();
-        hashMap.put("key1", "get");
-        Telemetry telemetry = new Telemetry(hashMap);
-        NullableValue nullableValue = telemetry.read("invalidKey1.invalidKey2");
-        assertEquals(null, nullableValue.value());
-        assertTrue(nullableValue.isNull());
-    }
-
-    @Test
-    public void shouldAddValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedMap = new HashMap<String, String>();
-        Telemetry telemetry = new Telemetry(hashMap);
-        telemetry.add("key", nestedMap);
-        assertEquals(nestedMap, telemetry.getMap().get("key"));
-    }
-
-    @Test
-    public void shouldOverrideValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedMap = new HashMap<String, String>();
-        HashMap<String, String> overrideNestedMap = new HashMap<String, String>();
-        hashMap.put("key", nestedMap);
-        Telemetry telemetry = new Telemetry(hashMap);
-        telemetry.add("key", overrideNestedMap);
-        assertEquals(overrideNestedMap, telemetry.getMap().get("key"));
-    }
-
-    @Test
-    public void shouldAddNestedValue() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedMap = new HashMap<String, String>();
-        hashMap.put("key", nestedMap);
-        HashMap<String, String> otherNestedMap = new HashMap<String, String>();
-        Telemetry telemetry = new Telemetry(hashMap);
-        telemetry.add("key.nested", otherNestedMap);
-        assertEquals(otherNestedMap, telemetry.read("key.nested").value());
-    }
-
-    @Test
-    public void shouldNotAddWhenKeysDoesNotExists() {
-        Map hashMap = new HashMap<String, Object>();
-        HashMap<String, String> nestedMap = new HashMap<String, String>();
-        Telemetry telemetry = new Telemetry(hashMap);
-        telemetry.add("invalidKey.nested", nestedMap);
-        assertFalse(telemetry.getMap().containsKey("invalidKey"));
-    }
-
-    @Test
-    public void shouldGetTheEts() {
-        double ets = 12334534;
-        Map hashMap = new HashMap<String, Object>();
-        hashMap.put("ets", ets);
-        HashMap<String, String> nestedMap = new HashMap<String, String>();
-        Telemetry telemetry = new Telemetry(hashMap);
-        try {
-            assertEquals(12334534, telemetry.getEts());
-        } catch (TelemetryReaderException e) {
-
-        }
-
-        assertFalse(telemetry.getMap().containsKey("invalidKey"));
-    }
-
-    @Test
-    public void shouldGetTheTimeStamp() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(telemetry.getAtTimestamp());
-    }
-
-    @Test
-    public void shouldGetTheSyncTS() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(telemetry.getSyncts());
-    }
-
-    @Test
-    public void shouldGetHashCodeValue() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(telemetry.hashCode());
-    }
-
-    @Test
-    public void shouldGetStringObject() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(telemetry.toString());
-    }
-
-    @Test
-    public void shouldGetTheMetaDataValue() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNotNull(telemetry.id());
-    }
-
-
-    @Test
-    public void CheckObjectsAreEqualOrNot() {
-        Telemetry telemetry1 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Telemetry telemetry2 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        String stringObj = new String();
-        Assert.assertTrue(telemetry1.equals(telemetry2));
-        Assert.assertFalse(telemetry1.equals(stringObj));
-    }
-
-    @Test
-    public void ShouldAddFieldIfNotPresent() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        telemetry.addFieldIfAbsent("error", "Invalid Key");
-        Assert.assertEquals(telemetry.read("error").value(), "Invalid Key");
-    }
-
-    @Test
-    public void ShouldReturnNullIfTheKeyIsNotPresent() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Assert.assertNull(telemetry.read("invalidKey").value());
-    }
-
-    @Test
-    public void shouldThrowError() {
-        Telemetry telemetry = new Telemetry(null);
-
-        try {
-            Assert.assertNull(telemetry.mustReadValue("invalidKey"));
-        } catch (Exception e) {
-            Assert.assertNotNull(e.getMessage());
-        }
-    }
-
-    @Test
-    public void shouldValidateTheNullableObject() {
-        Telemetry telemetry1 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        Telemetry telemetry2 = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        NullableValue<Object> eid1 = telemetry1.read("eid");
-        NullableValue<Object> eid2 = telemetry2.read("eid");
-        Assert.assertTrue(eid1.equals(eid2));
-        System.out.println(eid1.equals(eid2));
-        Assert.assertNotNull(eid1.hashCode());
-        Assert.assertNotNull(eid1.toString());
-    }
-
-    @Test
-    public void shouldGetTheDefaultValue() {
-        Telemetry telemetry = new Telemetry(null);
-        NullableValue<Object> eid1 = telemetry.read("eid");
-        Assert.assertEquals(eid1.valueOrDefault("START"), "START");
-    }
-
-    @Test
-    public void shouldGetTheActualValue() {
-        Telemetry telemetry = new Telemetry(EventFixture.getMap(EventFixture.IMPRESSION_EVENT));
-        NullableValue<Object> eid1 = telemetry.read("eid");
-        Assert.assertEquals(eid1.valueOrDefault("START"), "IMPRESSION");
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/events-router/pom.xml b/data-pipeline/events-router/pom.xml
deleted file mode 100644
index 392b2f1233..0000000000
--- a/data-pipeline/events-router/pom.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>events-router</artifactId>
-    <version>0.1.7</version>
-    <packaging>jar</packaging>
-    <name>EventsRouter</name>
-    <description>
-        Router job to route telemetry and summary events to separate topics
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-         <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/events-router/src/main/assembly/src.xml b/data-pipeline/events-router/src/main/assembly/src.xml
deleted file mode 100644
index e31a82cf30..0000000000
--- a/data-pipeline/events-router/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/events-router.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:events-router</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/events-router/src/main/config/events-router.properties b/data-pipeline/events-router/src/main/config/events-router.properties
deleted file mode 100644
index d122d557b4..0000000000
--- a/data-pipeline/events-router/src/main/config/events-router.properties
+++ /dev/null
@@ -1,91 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.EventsRouter
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__events_router_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.EventsRouterTask
-task.inputs=kafka.__env__.telemetry.denorm.valid
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-router.events.telemetry.route.topic=__env__.events.telemetry
-router.events.summary.route.topic=__env__.events.summary
-router.events.summary.route.events=ME_WORKFLOW_SUMMARY
-output.failed.topic.name=__env__.telemetry.failed
-output.duplicate.topic.name=__env__.telemetry.duplicate
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count,duplicate-event-count
-
-# dedup configurations
-
-dedup.enabled=true
-# redis
-#redis.host=localhost
-redis.host=__redis_host__
-#redis.port=6379
-redis.port=__redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.duplicationstore.id=8
-redis.database.key.expiry.seconds=86400
-
-dedup.exclude.eids=__dedup_exclude_eids__
-
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 8c40be6dab..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.task.EventsRouterConfig;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public void markFailure(String error, EventsRouterConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tr_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("metadata.tr_error", error);
-        telemetry.add("metadata.src", config.jobName());
-    }
-
-    public void markSkipped() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_dr_processed", false);
-        telemetry.add("flags.dd_dr_checksum_present", false);
-    }
-
-    public void markDuplicate() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.dd_dr_processed", false);
-        telemetry.add("flags.dd_dr_duplicate_event", true);
-    }
-
-}
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java
deleted file mode 100644
index f63cdabc53..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/service/EventsRouterService.java
+++ /dev/null
@@ -1,82 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.EventsRouterConfig;
-import org.ekstep.ep.samza.task.EventsRouterSink;
-import org.ekstep.ep.samza.task.EventsRouterSource;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import redis.clients.jedis.exceptions.JedisException;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-import static java.text.MessageFormat.format;
-
-public class EventsRouterService {
-
-	private static Logger LOGGER = new Logger(EventsRouterService.class);
-	private final DeDupEngine deDupEngine;
-	private final EventsRouterConfig config;
-
-	public EventsRouterService(DeDupEngine deDupEngine, EventsRouterConfig config) {
-
-		this.config = config;
-		this.deDupEngine = deDupEngine;
-	}
-
-	public void process(EventsRouterSource source, EventsRouterSink sink) {
-		Event event = null;
-		try {
-			event = source.getEvent();
-			if (config.isDedupEnabled() && isDupCheckRequired(event.eid())) {
-				String checksum = event.getChecksum();
-				if (!deDupEngine.isUniqueEvent(checksum)) {
-					LOGGER.info(event.id(), "DUPLICATE EVENT, CHECKSUM: {}", checksum);
-					event.markDuplicate();
-					sink.toDuplicateTopic(event);
-					return;
-				}
-				LOGGER.info(event.id(), "ADDING EVENT CHECKSUM TO STORE");
-				deDupEngine.storeChecksum(checksum);
-			} else {
-				LOGGER.info(event.id(), "SKIPPING THE DEDUP CHECK");
-			}
-			String eid = event.eid();
-			if(event.mid().contains("TRACE")){
-				SimpleDateFormat simple = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
-				String timeStamp  = simple.format(new Date());
-				event.updateTs(timeStamp);
-			}
-			String summaryRouteEventPrefix = this.config.getSummaryRouteEvents();
-			if (eid.startsWith(summaryRouteEventPrefix)) {
-				sink.toSummaryEventsTopic(event);
-			} else if (eid.startsWith("ME_")) {
-				sink.incrementSkippedCount(event);
-			} else {
-				sink.toTelemetryEventsTopic(event);
-			}
-		} catch (JedisException e) {
-			e.printStackTrace();
-			LOGGER.error(null, "Exception when retrieving data from redis: ", e);
-			deDupEngine.getRedisConnection().close();
-			throw e;
-		} catch (JsonSyntaxException e) {
-			e.printStackTrace();
-			LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-			sink.toMalformedTopic(source.getMessage());
-		} catch (Exception e) {
-			e.printStackTrace();
-			LOGGER.error(null,
-					format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:",
-							event),
-					e);
-			sink.toErrorTopic(event, e.getMessage());
-		}
-	}
-
-	private boolean isDupCheckRequired(String eid) {
-		return (config.excludedEids().isEmpty() || (null != eid && !(config.excludedEids().contains(eid))));
-	}
-}
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java
deleted file mode 100644
index 1a69d0cfc9..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterConfig.java
+++ /dev/null
@@ -1,86 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-import java.util.ArrayList;
-import java.util.List;
-
-public class EventsRouterConfig {
-
-    private final String JOB_NAME = "EventsRouter";
-
-    private String failedTopic;
-    private String duplicateTopic;
-    private String telemetryEventsRouteTopic;
-    private String summaryEventsRouteTopic;
-    private String summaryRouteEvents;
-    private String malformedTopic;
-    private boolean dedupEnabled;
-    private final int dupStore;
-    private int expirySeconds;
-    private List<String> excludedEids = new ArrayList<>();
-
-    public EventsRouterConfig(Config config) {
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        duplicateTopic = config.get("output.duplicate.topic.name", "telemetry.duplicate");
-        telemetryEventsRouteTopic = config.get("router.events.telemetry.route.topic", "events.telemetry");
-        summaryRouteEvents = config.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY");
-        summaryEventsRouteTopic = config.get("router.events.summary.route.topic", "events.summary");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        dedupEnabled = config.getBoolean("dedup.enabled", true);
-        dupStore = config.getInt("redis.database.duplicationstore.id", 8);
-        expirySeconds = config.getInt("redis.database.key.expiry.seconds", 28800);
-        if (!config.get("dedup.exclude.eids", "").isEmpty()) {
-            excludedEids = config.getList("dedup.exclude.eids", new ArrayList<>());
-        }
-        else {
-            excludedEids = new ArrayList<>();
-        }
-    }
-
-    public String getTelemetryEventsRouteTopic() {
-        return telemetryEventsRouteTopic;
-    }
-
-
-    public String getSummaryEventsRouteTopic() {
-        return summaryEventsRouteTopic;
-    }
-
-
-    public String getSummaryRouteEvents() {
-        return this.summaryRouteEvents;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String duplicateTopic() {
-        return duplicateTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public Boolean isDedupEnabled() {
-        return dedupEnabled;
-    }
-
-    public int dupStore() {
-        return dupStore;
-    }
-
-    public int expirySeconds() {
-        return expirySeconds;
-    }
-
-    public List<String> excludedEids() {
-        return excludedEids;
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java
deleted file mode 100644
index fa34845910..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSink.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class EventsRouterSink extends BaseSink {
-
-	// private JobMetrics metrics;
-	private EventsRouterConfig config;
-	
-	public EventsRouterSink(MessageCollector collector, JobMetrics metrics, EventsRouterConfig config) {
-		super(collector, metrics);
-		// this.metrics = metrics;
-		this.config = config;
-	}
-
-	public void toTelemetryEventsTopic(Event event) {
-		toTopic(config.getTelemetryEventsRouteTopic(), event.did(), event.getJson());
-		metrics.incSuccessCounter();
-	}
-
-	public void toErrorTopic(Event event, String errorMessage) {
-		event.markFailure(errorMessage, config);
-		toTopic(config.failedTopic(), event.did(), event.getJson());
-		metrics.incErrorCounter();
-	}
-
-	public void toDuplicateTopic(Event event) {
-		toTopic(config.duplicateTopic(), event.did(), event.getJson());
-		metrics.incDuplicateCounter();
-	}
-
-	public void toMalformedTopic(String message) {
-		toTopic(config.malformedTopic(), null, message);
-		metrics.incErrorCounter();
-	}
-
-	public void toSummaryEventsTopic(Event event) {
-		toTopic(config.getSummaryEventsRouteTopic(), event.did(), event.getJson());
-		metrics.incSuccessCounter();
-	}
-
-	public void incrementSkippedCount(Event event) {
-		metrics.incSkippedCounter();
-	}
-
-}
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java
deleted file mode 100644
index ad150c7f5e..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class EventsRouterSource {
-    static Logger LOGGER = new Logger(EventsRouterSource.class);
-
-    private IncomingMessageEnvelope envelope;
-
-    public EventsRouterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-
-}
diff --git a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java b/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java
deleted file mode 100644
index 2c137fb5bf..0000000000
--- a/data-pipeline/events-router/src/main/java/org/ekstep/ep/samza/task/EventsRouterTask.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.service.EventsRouterService;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class EventsRouterTask extends BaseSamzaTask {
-
-	static Logger LOGGER = new Logger(EventsRouterTask.class);
-	private EventsRouterConfig config;
-	private JobMetrics metrics;
-	private EventsRouterService service;
-
-	public EventsRouterTask(DeDupEngine deDupEngine, Config config, TaskContext context) {
-		init(config, context, deDupEngine);
-	}
-
-	public EventsRouterTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		
-		this.config = new EventsRouterConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		init(config, context, null);
-	}
-
-	private void init(Config config, TaskContext context, DeDupEngine deDupEngine) {
-		this.config = new EventsRouterConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		deDupEngine = deDupEngine == null ?
-				new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine;
-		service = new EventsRouterService(deDupEngine, this.config);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator)
-			throws Exception {
-
-		EventsRouterSink sink = new EventsRouterSink(collector, metrics, config);
-		EventsRouterSource source = new EventsRouterSource(envelope);
-		service.process(source, sink);
-	}
-	
-}
diff --git a/data-pipeline/events-router/src/main/resources/log4j.xml b/data-pipeline/events-router/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/events-router/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index d783838071..0000000000
--- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,327 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-
-import java.util.Map;
-
-public class EventFixture {
-
-	public static final String WORKFLOW_SUMMARY_EVENT = "{\n" +
-			"  \"eid\": \"ME_WORKFLOW_SUMMARY\",\n" +
-			"  \"ets\": 1548040103586,\n" +
-			"  \"syncts\": 1547991371195,\n" +
-			"  \"mid\": \"837C68F76FB02D338E39DFBEE1095E3B\",\n" +
-			"  \"context\": {\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"AnalyticsDataPipeline\",\n" +
-			"      \"ver\": \"1.0\",\n" +
-			"      \"model\": \"WorkflowSummarizer\"\n" +
-			"    },\n" +
-			"    \"granularity\": \"SESSION\",\n" +
-			"    \"date_range\": {\n" +
-			"      \"from\": 1547991074190,\n" +
-			"      \"to\": 1547991170529\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"dimensions\": {\n" +
-			"    \"did\": \"e57185808603966cb58e18647bfc5f0de4e4d820\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"prod.diksha.app\",\n" +
-			"      \"ver\": \"2.1.8\",\n" +
-			"      \"pid\": \"sunbird.app\"\n" +
-			"    },\n" +
-			"    \"sid\": \"f962116f-3f84-446b-964c-fae92a15fefd\",\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"type\": \"session\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"eks\": {\n" +
-			"      \"interact_events_per_min\": 3.74,\n" +
-			"      \"start_time\": 1547991074190,\n" +
-			"      \"end_time\": 1547991170529,\n" +
-			"      \"events_summary\": [\n" +
-			"        { \"id\": \"START\", \"count\": 3 },\n" +
-			"        { \"id\": \"IMPRESSION\", \"count\": 5 },\n" +
-			"        { \"id\": \"END\", \"count\": 3 }\n" +
-			"      ],\n" +
-			"      \"page_summary\": [\n" +
-			"        {\n" +
-			"          \"id\": \"collection-detail\",\n" +
-			"          \"type\": \"detail\",\n" +
-			"          \"env\": \"home\",\n" +
-			"          \"time_spent\": 13.45,\n" +
-			"          \"visit_count\": 1\n" +
-			"        },\n" +
-			"        {\n" +
-			"          \"id\": \"library\",\n" +
-			"          \"type\": \"search\",\n" +
-			"          \"env\": \"home\",\n" +
-			"          \"time_spent\": 28.09,\n" +
-			"          \"visit_count\": 1\n" +
-			"        }\n" +
-			"      ],\n" +
-			"      \"time_diff\": 96.34,\n" +
-			"      \"telemetry_version\": \"3.0\",\n" +
-			"      \"env_summary\": [\n" +
-			"        {\n" +
-			"          \"env\": \"home\",\n" +
-			"          \"time_spent\": 41.54,\n" +
-			"          \"count\": 1\n" +
-			"        }\n" +
-			"      ],\n" +
-			"      \"time_spent\": 96.36\n" +
-			"    }\n" +
-			"  }\n" +
-			"}";
-
-	public static final String ERROR_EVENT = "{\n" +
-			"  \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" +
-			"  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-			"  \"eid\": \"ERROR\",\n" +
-			"  \"gdata\": {\n" +
-			"    \"id\": \"genie.android\",\n" +
-			"    \"ver\": \"2.2.15\"\n" +
-			"  },\n" +
-			"  \"pdata\": {\n" +
-			"    \"id\": \"genie\",\n" +
-			"    \"ver\": \"2.0\"\n" +
-			"  },\n" +
-			"  \"sid\": \"\",\n" +
-			"  \"ets\": 1454064092546,\n" +
-			"  \"uid\": \"\",\n" +
-			"  \"ver\": \"2.2\",\n" +
-			"  \"channel\": \"in.ekstep\",\n" +
-			"  \"cdata\": [\n" +
-			"    {\n" +
-			"      \"id\": \"correlationid\",\n" +
-			"      \"type\": \"correlationtype\"\n" +
-			"    }\n" +
-			"  ]\n" +
-			"}";
-	public static final String UNPARSABLE_START_EVENT = "{\n" +
-			"  \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" +
-			"  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-			"  \"edata\": {\n" +
-			"    \"eks\": {\n" +
-			"      \"mode\": \"WIFI\",\n" +
-			"      \"ver\": \"12\",\n" +
-			"      \"size\": 12.67,\n" +
-			"      \"err\": \"\",\n" +
-			"      \"referrer\": [\n" +
-			"        {\n" +
-			"          \"action\": \"INSTALL\",\n" +
-			"          \"utmsource\": \"Ekstep\",\n" +
-			"          \"utmmedium\": \"Portal\",\n" +
-			"          \"utmterm\": \"December 2016\",\n" +
-			"          \"utmcontent\": \"Ramayana\",\n" +
-			"          \"utmcampaign\": \"Epics of India\"\n" +
-			"        }\n" +
-			"      ]\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"eid\": \"START\",\n" +
-			"  \"gdata\": {\n" +
-			"    \"id\": \"genie.android\",\n" +
-			"    \"ver\": \"1.0\"\n" +
-			"  },\n" +
-			"  \"sid\": \"\",\n" +
-			"  \"ets\": 1454064092546,\n" +
-			"  \"uid\": \"\",\n" +
-			"  \"ver\": \"2.0\",\n" +
-			"  \"cdata\": [\n" +
-			"    {\n" +
-			"      \"id\": \"correlationid\",\n" +
-			"      \"type\": \"correlationtype\"\n" +
-			"    ";
-	public static final String START_EVENT = "{\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"3ae5dc29-17ea-4465-800e-a4361b9d9604\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"type\": \"app\"\n" +
-			"  },\n" +
-			"  \"eid\": \"START\",\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"ets\": 1548720004773,\n" +
-			"  \"context\": {\n" +
-			"    \"pdata\": {\n" +
-			"      \"ver\": \"2.1.30\",\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"id\": \"prod.diksha.app\"\n" +
-			"    },\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"env\": \"home\",\n" +
-			"    \"did\": \"28b3576420784da6a7b556c1966c759fc793ac0d\",\n" +
-			"    \"sid\": \"042d7f61-225b-4e3f-9fe4-93f373368d8f\",\n" +
-			"    \"cdata\": []\n" +
-			"  },\n" +
-			"  \"mid\": \"b8b0f2ca-ce0a-427d-9c47-195eb5424071\",\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"\",\n" +
-			"    \"type\": \"\",\n" +
-			"    \"version\": \"\"\n" +
-			"  },\n" +
-			"  \"tags\": [],\n" +
-			"  \"syncts\": 1548720010819,\n" +
-			"  \"@timestamp\": \"2019-01-29T00:00:10.819Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-	public static final String EVENT_WITH_NULL_EID = "{\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"3ae5dc29-17ea-4465-800e-a4361b9d9604\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"type\": \"app\"\n" +
-			"  },\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"ets\": 1548720004773,\n" +
-			"  \"context\": {\n" +
-			"    \"pdata\": {\n" +
-			"      \"ver\": \"2.1.30\",\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"id\": \"prod.diksha.app\"\n" +
-			"    },\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"env\": \"home\",\n" +
-			"    \"did\": \"28b3576420784da6a7b556c1966c759fc793ac0d\",\n" +
-			"    \"sid\": \"042d7f61-225b-4e3f-9fe4-93f373368d8f\",\n" +
-			"    \"cdata\": []\n" +
-			"  },\n" +
-			"  \"mid\": \"b8b0f2ca-ce0a-427d-9c47-195eb5424071\",\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"\",\n" +
-			"    \"type\": \"\",\n" +
-			"    \"version\": \"\"\n" +
-			"  },\n" +
-			"  \"tags\": [],\n" +
-			"  \"syncts\": 1548720010819,\n" +
-			"  \"@timestamp\": \"2019-01-29T00:00:10.819Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-	public static final String ANY_STRING = "Hey Samza, Whats Up?";
-	public static final String EMPTY_JSON = "{}";
-
-	public static final String LOG_EVENT = "{\n"+
-			"   \"@timestamp\":\"2019-03-20T00:00:01.176Z\",\n"+
-			"\"actor\":{\n"+
-			"\"id\":\"0b251080-3230-415e-a593-ab7c1fac7ae3\",\n"+
-			"\"type\":\"User\"\n"+
-			"},\n"+
-			"\"context\":{\n"+
-			"\"cdata\":[\n"+
-
-			"],\n"+
-			"\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\n"+
-			/*"\"did\":\"923c675274cfbf19fd0402fe4d2c37afd597f0ab\",\n"+*/
-			"\"env\":\"home\",\n"+
-			"\"pdata\":{\n"+
-			"\"id\":\"prod.diksha.app\",\n"+
-			"\"pid\":\"sunbird.app\",\n"+
-			"\"ver\":\"2.1.45\"\n"+
-			"},\n"+
-			"\"sid\":\"57b5b7ea-93c5-48d6-ba51-f5f9a3570ffe\"\n"+
-			"},\n"+
-			"\"edata\":{\n"+
-			"\"level\":\"INFO\",\n"+
-			"\"message\":\"content-detail\",\n"+
-			"\"params\":[\n"+
-			"{\n"+
-			"\"PopupType\":\"automatic\"\n"+
-			"}\n"+
-			"],\n"+
-			"\"type\":\"view\"\n"+
-			"},\n"+
-			"\"eid\":\"LOG\",\n"+
-			"\"ets\":1.553039987481E12,\n"+
-			"\"flags\":{\n"+
-			"\"dd_processed\":true,\n"+
-			"\"tv_processed\":true\n"+
-			"},\n"+
-			"\"mid\":\"ca17e5bd-71d4-487a-92cd-0fb377e7a591\",\n"+
-			"\"syncts\":1.553040001176E12,\n"+
-			"\"tags\":[\n"+
-			"],\n"+
-			"\"type\":\"events\",\n"+
-			"\"ver\":\"3.0\"\n"+
-			"      }";
-	public static final String TELEMETRY_ERROR_EVENT = "{"+
-			"   \"eid\":\"ERROR\","+
-			"   \"ets\":%s,"+
-			"   \"ver\":\"3.0\","+
-			"   \"mid\":\"LP.1553040097857.bf0e4e15-014e-4a22-ba00-e02ff3c38784\","+
-			"   \"actor\":{"+
-			"      \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\","+
-			"      \"type\":\"System\""+
-			"   },"+
-			"   \"context\":{"+
-			"      \"channel\":\"01235953109336064029450\","+
-			"      \"pdata\":{"+
-			"         \"id\":\"prod.ntp.learning.platform\","+
-			"         \"pid\":\"learning-service\","+
-			"         \"ver\":\"1.0\""+
-			"      },"+
-			"      \"env\":\"framework\""+
-			"   },"+
-			"   \"edata\":{"+
-			"      \"err\":\"ERR_DATA_NOT_FOUND\","+
-			"      \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\","+
-			"      \"errtype\":\"system\""+
-			"   },"+
-			"   \"flags\":{"+
-			"      \"tv_processed\":true,"+
-			"      \"dd_processed\":true"+
-			"   },"+
-			"   \"type\":\"events\","+
-			"   \"syncts\":1.553040098435E12,"+
-			"   \"@timestamp\":\"2019-03-20T00:01:38.435Z\""+
-			"}";
-
-	public static final String TRACE_EVENT = "{"+
-			"   \"eid\":\"TRACE\","+
-			"   \"ets\":%s,"+
-			"   \"ver\":\"3.0\","+
-			"   \"mid\":\"TRACE.bf0e4e15-014e-4a22-ba00-e02ff3c38784\","+
-			"   \"actor\":{"+
-			"      \"id\":\"e85bcfb5-a8c2-4e65-87a2-0ebb43b45f01\","+
-			"      \"type\":\"System\""+
-			"   },"+
-			"   \"context\":{"+
-			"      \"channel\":\"01235953109336064029450\","+
-			"      \"pdata\":{"+
-			"         \"id\":\"prod.ntp.learning.platform\","+
-			"         \"pid\":\"learning-service\","+
-			"         \"ver\":\"1.0\""+
-			"      },"+
-			"      \"env\":\"framework\""+
-			"   },"+
-			"   \"edata\":{"+
-			"      \"err\":\"ERR_DATA_NOT_FOUND\","+
-			"      \"stacktrace\":\"ERR_DATA_NOT_FOUND: Data not found with id : null\n\tat\","+
-			"      \"errtype\":\"system\""+
-			"   },"+
-			"   \"flags\":{"+
-			"      \"tv_processed\":true,"+
-			"      \"dd_processed\":true"+
-			"   },"+
-			"   \"type\":\"events\","+
-			"   \"syncts\":1.553040098435E12,"+
-			"   \"@timestamp\":\"2019-03-20T00:01:38.435Z\""+
-			"}";
-
-	public static final String WORKFLOW_USAGE_EVENT = "{\"eid\":\"ME_WORKFLOW_USAGE_SUMMARY\",\"ets\":1580174594135,\"syncts\":1580112438526,\"ver\":\"2.1\",\"mid\":\"7FA1DEB21833F00EE8AD66FF8AFF4589\",\"uid\":\"\",\"channel\":\"\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"WorkFlowUsageSummarizer\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1580108039040,\"to\":1580112356069}},\"dimensions\":{\"uid\":\"04b88827-87b1-40fd-a662-b1b97068d127\",\"did\":\"all\",\"pdata\":{\"id\":\"prod.diksha.app\",\"ver\":\"2.3.162\",\"pid\":\"sunbird.app\"},\"tag\":\"all\",\"period\":20200127,\"content_id\":\"all\",\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"type\":\"resource\",\"mode\":\"play\",\"content_type\":\"all\"},\"edata\":{\"eks\":{\"total_users_count\":0,\"total_devices_count\":1,\"total_content_count\":4,\"avg_ts_session\":4.88,\"total_sessions\":5,\"avg_interactions_min\":24.58,\"total_interactions\":10,\"avg_pageviews\":0.0,\"total_ts\":24.41,\"total_pageviews_count\":0}}}";
-	public static Map<String, Object> getMap(String message) {
-		return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-	}
-
-}
\ No newline at end of file
diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java
deleted file mode 100644
index a22a07d379..0000000000
--- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-import java.util.Map;
-
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-
-public class EventTest {
-	
-	@Test
-	public void shouldReturnMid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT));
-		Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid());
-	}
-
-	@Test
-	public void shouldReturnEid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.WORKFLOW_SUMMARY_EVENT));
-		Assert.assertEquals("ME_WORKFLOW_SUMMARY", event.eid());
-	}
-
-	@Test(expected = JsonSyntaxException.class)
-	public void shouldThrowException() {
-		new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT,
-				new TypeToken<Map<String, Object>>() {
-				}.getType());
-	}
-
-	@Test
-	public void shouldAddSkipFlags() {
-		String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-		Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-		event.markSkipped();
-		Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-		Assert.assertNotNull(flagData);
-		Assert.assertEquals(flagData.get("dd_dr_processed"), false);
-		Assert.assertEquals(flagData.get("dd_dr_checksum_present"), false);
-	}
-}
diff --git a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java b/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java
deleted file mode 100644
index f10d5d4a45..0000000000
--- a/data-pipeline/events-router/src/test/java/org/ekstep/ep/samza/task/EventsRouterTaskTest.java
+++ /dev/null
@@ -1,231 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.fiftyonred.mock_jedis.MockJedis;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class EventsRouterTaskTest {
-
-	private static final String TELEMETRY_EVENTS_TOPIC = "events.telemetry";
-	private static final String SUMMARY_EVENTS_TOPIC = "events.summary";
-	private static final String FAILED_TOPIC = "telemetry.failed";
-	private static final String MALFORMED_TOPIC = "telemetry.malformed";
-	private static final String DUPLICATE_TOPIC = "telemetry.duplicate";
-
-	private MessageCollector collectorMock;
-	private TaskContext contextMock;
-	private MetricsRegistry metricsRegistry;
-	private Counter counter;
-	private TaskCoordinator coordinatorMock;
-	private IncomingMessageEnvelope envelopeMock;
-	private Config configMock;
-	private EventsRouterTask eventsRouterTask;
-	private DeDupEngine deDupEngineMock;
-	private RedisConnect redisConnectMock;
-	private Jedis jedisMock = new MockJedis("duplicationtest");
-	private int dupStoreId = 1;
-
-	@Before
-	public void setUp() {
-		collectorMock = mock(MessageCollector.class);
-		contextMock = Mockito.mock(TaskContext.class);
-		metricsRegistry = Mockito.mock(MetricsRegistry.class);
-		counter = Mockito.mock(Counter.class);
-		coordinatorMock = mock(TaskCoordinator.class);
-		envelopeMock = mock(IncomingMessageEnvelope.class);
-		configMock = Mockito.mock(Config.class);
-
-		redisConnectMock = mock(RedisConnect.class);
-		deDupEngineMock = mock(DeDupEngine.class);
-		stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-		stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-		stub(configMock.get("router.events.telemetry.route.topic", TELEMETRY_EVENTS_TOPIC)).toReturn(TELEMETRY_EVENTS_TOPIC);
-		stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-		stub(configMock.get("router.events.summary.route.topic", SUMMARY_EVENTS_TOPIC)).toReturn(SUMMARY_EVENTS_TOPIC);
-		stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-		stub(configMock.get("output.duplicate.topic.name", DUPLICATE_TOPIC)).toReturn(DUPLICATE_TOPIC);
-		stub(configMock.getBoolean("dedup.enabled", true)).toReturn(true);
-		when(configMock.get("dedup.exclude.eids", "")).thenReturn("LOG");
-		stub(configMock.getList("dedup.exclude.eids", new ArrayList<>())).toReturn(Arrays.asList("LOG","ERROR"));
-
-		stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-		stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-		stub(envelopeMock.getOffset()).toReturn("2");
-		stub(envelopeMock.getSystemStreamPartition())
-				.toReturn(new SystemStreamPartition("kafka", "telemetry.denorm.valid", new Partition(1)));
-
-	}
-
-	@Test
-	public void shouldRouteSummaryEventsToSummaryTopic() throws Exception {
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_SUMMARY_EVENT);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUMMARY_EVENTS_TOPIC)));
-	}
-
-	@Test
-	public void shouldRouteTelemetryEventsToTelemetryTopic() throws Exception {
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC)));
-
-	}
-
-	@Test
-	public void shouldStoreChecksumForUniqueEvents() throws Exception {
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		DeDupEngine deDupEngine = new DeDupEngine(redisConnectMock, dupStoreId,60);
-		boolean isUnique = deDupEngine.isUniqueEvent("677009782");
-		deDupEngine.getRedisConnection();
-		deDupEngine.storeChecksum("678998676");
-
-		assertEquals(isUnique, true);
-	}
-
-	@Test
-	public void shouldMarkEventFailureIfNullEid() throws Exception {
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_NULL_EID);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception {
-		when(configMock.get("dedup.exclude.eids", "")).thenReturn("");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendEventToDuplicateTopic() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(false);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(deDupEngineMock, times(1)).isUniqueEvent(any());
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), DUPLICATE_TOPIC)));
-	}
-
-	public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-		return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-				assertEquals("kafka", systemStream.getSystem());
-				assertEquals(stream, systemStream.getStream());
-				return true;
-			}
-		};
-	}
-
-	@Test
-	public void shouldRouteLogEventsToTelemetryEventsTopic() throws Exception {
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(deDupEngineMock, times(0)).isUniqueEvent(any());
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC)));
-	}
-
-	@Test
-	public void shouldRouteErrorEventsToTelemetryEventsTopic() throws Exception {
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.TELEMETRY_ERROR_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(deDupEngineMock, times(0)).isUniqueEvent(any());
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC)));
-	}
-
-	@Test
-	public void shouldStampTimeStampforTraceEvent() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.TRACE_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-	}
-
-	@Test
-	public void shouldSkipsSummaryEvents() throws Exception {
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_USAGE_EVENT);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-	}
-
-	@Test(expected = JedisException.class)
-	public void shouldCatchRedisException() throws Exception{
-
-		stub(configMock.get("router.events.summary.route.events", "ME_WORKFLOW_SUMMARY")).toReturn("ME_WORKFLOW_SUMMARY");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.WORKFLOW_USAGE_EVENT);
-		Jedis jedismock = mock(Jedis.class);
-		when(deDupEngineMock.isUniqueEvent(anyString())).thenReturn(true);
-		when(deDupEngineMock.getRedisConnection()).thenReturn(jedismock);
-		eventsRouterTask = new EventsRouterTask(deDupEngineMock, configMock, contextMock);
-		when(deDupEngineMock.isUniqueEvent(any())).thenThrow(JedisException.class);
-		eventsRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-
-	}
-}
diff --git a/data-pipeline/pom.xml b/data-pipeline/pom.xml
deleted file mode 100644
index 2da519fb5f..0000000000
--- a/data-pipeline/pom.xml
+++ /dev/null
@@ -1,94 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
-                             http://maven.apache.org/maven-v4_0_0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-
-	<prerequisites>
-		<maven>3.0.0</maven>
-	</prerequisites>
-
-	<groupId>org.ekstep.ecosystem</groupId>
-	<artifactId>jobs</artifactId>
-	<version>0.0.1</version>
-	<packaging>pom</packaging>
-	<name>Ekstep Samza</name>
-	<description>
-    	EkStep Samza for stream processing within Ecosystem
-  	</description>
-	<url>https://ekstep.org/</url>
-
-	<modules>
-		<module>ep-core</module>
-		<module>distribution</module>
-		<module>ep-telemetry-reader</module>
-		<module>telemetry-validator</module>
-		<module>telemetry-extractor</module>
-		<module>de-duplication</module>
-		<module>derived-de-duplication</module>
-		<module>telemetry-router</module>
-		<module>telemetry-redacter</module>
-		<module>telemetry-location-updater</module>
-		<module>de-normalization</module>
-		<module>druid-events-validator</module>
-		<module>events-router</module>
-		<module>device-profile-updater</module>
-		<module>assessment-aggregator</module>
-		<module>user-cache-updater</module>
-		<module>content-cache-updater</module>
-		<module>share-events-flattener</module>
-	</modules>
-
-	<properties>
-		<!-- maven specific properties -->
-		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<samza.version>0.14.1</samza.version>
-		<scala.version>2.11</scala.version>
-		<kafka.version>1.1.0</kafka.version>
-		<jackson-jaxrs.version>1.9.13</jackson-jaxrs.version>
-		<samza.log.dir>/tmp</samza.log.dir>
-		<samza.container.name>test-container</samza.container.name>
-	</properties>
-
-	<repositories>
-		<repository>
-			<id>my-local-repo</id>
-			<url>file://${user.home}/.m2/repository</url>
-		</repository>
-		<repository>
-			<id>apache-releases</id>
-			<url>https://repository.apache.org/content/groups/public</url>
-		</repository>
-		<repository>
-			<id>scala-tools.org</id>
-			<name>Scala-tools Maven2 Repository</name>
-			<url>https://oss.sonatype.org/content/groups/scala-tools</url>
-		</repository>
-	</repositories>
-
-	<pluginRepositories>
-		<pluginRepository>
-			<id>scala-tools.org</id>
-			<name>Scala-tools Maven2 Repository</name>
-			<url>http://scala-tools.org/repo-releases</url>
-		</pluginRepository>
-	</pluginRepositories>
-
-	<build>
-		<pluginManagement>
-			<plugins>
-				<plugin>
-					<groupId>org.apache.maven.plugins</groupId>
-					<artifactId>maven-compiler-plugin</artifactId>
-					<version>3.1</version>
-					<configuration>
-						<source>1.8</source>
-						<target>1.8</target>
-					</configuration>
-				</plugin>
-			</plugins>
-		</pluginManagement>
-	</build>
-
-</project>
diff --git a/data-pipeline/share-events-flattener/pom.xml b/data-pipeline/share-events-flattener/pom.xml
deleted file mode 100644
index 89bd5385a0..0000000000
--- a/data-pipeline/share-events-flattener/pom.xml
+++ /dev/null
@@ -1,172 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>share-events-flattener</artifactId>
-    <version>0.0.1</version>
-    <packaging>jar</packaging>
-    <name>Share Events Flattener</name>
-    <description>
-        Events Flattener job to flatten the share events.
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/share-events-flattener/src/main/assembly/src.xml b/data-pipeline/share-events-flattener/src/main/assembly/src.xml
deleted file mode 100644
index 8335b290c5..0000000000
--- a/data-pipeline/share-events-flattener/src/main/assembly/src.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/share-events-flattener.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:share-events-flattener</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties b/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties
deleted file mode 100644
index 4a67cb58d9..0000000000
--- a/data-pipeline/share-events-flattener/src/main/config/share-events-flattener.properties
+++ /dev/null
@@ -1,67 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.ShareEventsFlattener
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__events_flattener_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.EventsFlattenerTask
-task.inputs=kafka.__env__.telemetry.share
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.success.topic.name=__env__.telemetry.sink
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,error-message-count
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 4ea73764e4..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,45 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Events;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public void markSuccess() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.ef_processed", true);
-        telemetry.add("type", "events");
-    }
-
-    public void updateEventObjectKey(String objId, String type, String version, String rollupLevel1) {
-        telemetry.addFieldIfAbsent("object", new HashMap<String, String>());
-        telemetry.add("object.id", objId);
-        telemetry.add("object.type", type);
-        telemetry.add("object.version", version);
-        telemetry.add("object.rollup.l1", rollupLevel1);
-    }
-
-    public void updatedEventEdata(String edataType, Long size) {
-        telemetry.add("edata.type", edataType);
-        telemetry.add("edata.size", size);
-    }
-
-    public void updateMid(String mid){
-        telemetry.add("mid", mid);
-    }
-
-    public void removeItems() {
-        telemetry.add("edata.items", null);
-    }
-
-    public void renameEventIdTo(String eid) {
-        telemetry.add("eid", eid);
-    }
-
-}
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java
deleted file mode 100644
index 474e94ea08..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/service/EventsFlattenerService.java
+++ /dev/null
@@ -1,145 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.EventsFlattenerConfig;
-import org.ekstep.ep.samza.task.EventsFlattenerSink;
-import org.ekstep.ep.samza.task.EventsFlattenerSource;
-
-import java.lang.reflect.Type;
-import java.math.BigDecimal;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.UUID;
-
-public class EventsFlattenerService {
-
-    private static Logger LOGGER = new Logger(EventsFlattenerService.class);
-    private final EventsFlattenerConfig config;
-    private static String IMPORT_KEY = "import";
-    private static String DOWNLOAD_KEY = "download";
-    private static String FLATTEN_EVENT_NAME = "SHARE_ITEM";
-
-    /**
-     * Constructor of the EventFlattenService
-     *
-     * @param config - Configurations of the EventsFlatten Samza job.
-     */
-    public EventsFlattenerService(EventsFlattenerConfig config) {
-        this.config = config;
-    }
-
-    /**
-     * Listening to "telemetry.share" topic, Assuming only share events should produce
-     * From this "telemetry.share" topic.
-     *
-     * @param source - Producer - "telemetry.share" topic
-     * @param sink   - Consumer - "telemetry.sink"
-     */
-    public void process(EventsFlattenerSource source, EventsFlattenerSink sink) {
-        Event event = source.getEvent();
-        // Flattening the "SHARE" Event to Multiple "SHARE_ITEM" Events
-        this.toFlatten(event, getClonedEventObject(event), sink);
-        // Adding Original "SHARE" Events to success topic
-        sink.toSuccessTopic(event);
-    }
-
-    /**
-     * Method to flatten the SHARE EVENT Object to multiple share event.
-     * Events flattening constraints
-     * <p>
-     * ================= Constraints===============================
-     * 1. If the Item list object as params.transfers = 0 then edata.type should be "download" else do not modify the edata.type value.
-     * 2. If the Item list object as params.transfers > 0 then edata.type should be "import" else do not modify the edta.type value.
-     * 3. If the Item list object as params.transfers = null then do not update the edata.type (Keep the orginal value ie., dir, type)
-     * 4. If the share event has object then move the object data to rollup l1, share event item.id, item.typ and item.ver should be in object properties
-     * ===============================================================
-     *
-     * @param orginalEvent - SHARE Event object.
-     * @param sink         - Object to push the event to kafka sink
-     */
-    private void toFlatten(Event orginalEvent, Event clonedEvent, EventsFlattenerSink sink) {
-        String objectId = null;
-        String eDataType = clonedEvent.edataType();
-        Gson gson = new Gson();
-        Type type = new TypeToken<List<Map<String, String>>>() {
-        }.getType();
-
-        List<Map<String, Object>> items = clonedEvent.edataItems();
-        if (clonedEvent.objectFieldsPresent()) {
-            objectId = clonedEvent.objectID();
-        }
-        String itemId = null;
-        String itemType = null;
-        String itemVerion = null;
-
-        for (Map<String, Object> item : items) {
-            if (item.get("id") != null) {
-                itemId = item.get("id").toString();
-            }
-            if (item.get("type") != null) {
-                itemType = item.get("type").toString();
-            }
-            if (item.get("ver") != null) {
-                itemVerion = item.get("ver").toString();
-            }
-            clonedEvent.updateEventObjectKey(itemId, itemType, itemVerion, objectId);
-            Object itemParams = item.get("params");
-            if (itemParams != null) {
-                List<Map<String, String>> param = gson.fromJson(gson.toJson(itemParams), type);
-                String paramTransfer = this.getValue(param, "transfers");
-                Long paramSize = null;
-                if (this.getValue(param, "size") != null && !this.getValue(param, "size").isEmpty()) {
-                    paramSize = new BigDecimal(Objects.requireNonNull(this.getValue(param, "size"))).longValue();
-                }
-                if (paramTransfer != null) {
-                    if (Double.parseDouble(paramTransfer) == 0) {
-                        eDataType = DOWNLOAD_KEY;
-                    }
-                    if (Double.parseDouble(paramTransfer) > 0) {
-                        eDataType = IMPORT_KEY;
-                    }
-                    clonedEvent.updatedEventEdata(eDataType, paramSize);
-                } else {
-                    clonedEvent.updatedEventEdata(orginalEvent.edataType(), paramSize);
-                }
-            } else {
-                clonedEvent.updatedEventEdata(orginalEvent.edataType(), null);
-            }
-            clonedEvent.renameEventIdTo(FLATTEN_EVENT_NAME);
-            clonedEvent.removeItems();
-            // Adding "SHARE_ITEM" Events to success topic
-            clonedEvent.updateMid(this.getMid());
-            sink.toSuccessTopic(clonedEvent);
-        }
-    }
-
-    /**
-     * Method to clone the telemetry event object data.
-     *
-     * @param event Event object
-     * @return Duplicated Event object.
-     */
-    private Event getClonedEventObject(Event event) {
-        String message = event.getJson();
-        return new Event((Map<String, Object>) new Gson().fromJson(message, Map.class));
-    }
-
-
-    private String getValue(final List<Map<String, String>> paramsList, final String name) {
-        for (Map<String, String> p : paramsList) {
-            if (p.get(name) != null) {
-                return p.get(name);
-            }
-        }
-        return null;
-    }
-
-    private String getMid() {
-        return "SHARE_ITEM:" + UUID.randomUUID();
-    }
-
-}
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java
deleted file mode 100644
index 0573e6522d..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerConfig.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-public class EventsFlattenerConfig {
-
-    private final String JOB_NAME = "SHARE_EVENTS_FLATTENER";
-
-    private String successEventTopic;
-
-    public EventsFlattenerConfig(Config config) {
-        successEventTopic = config.get("output.success.topic.name", "telemetry.sink");
-    }
-
-    public String getSuccessTopic() {
-        return successEventTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java
deleted file mode 100644
index 68135a44b5..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSink.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class EventsFlattenerSink extends BaseSink {
-
-    private EventsFlattenerConfig config;
-
-    public EventsFlattenerSink(MessageCollector collector, JobMetrics metrics, EventsFlattenerConfig config) {
-        super(collector, metrics);
-        this.config = config;
-    }
-
-    public void toSuccessTopic(Event event) {
-        event.markSuccess();
-        toTopic(config.getSuccessTopic(), event.did(), event.getJson());
-        metrics.incSuccessCounter();
-    }
-
-}
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java
deleted file mode 100644
index 61645f616a..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerSource.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.domain.Event;
-
-import java.util.Map;
-
-public class EventsFlattenerSource {
-
-    private IncomingMessageEnvelope envelope;
-
-    public EventsFlattenerSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-}
diff --git a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java b/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java
deleted file mode 100644
index b52d767fb0..0000000000
--- a/data-pipeline/share-events-flattener/src/main/java/org/ekstep/ep/samza/task/EventsFlattenerTask.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.EventsFlattenerService;
-
-public class EventsFlattenerTask extends BaseSamzaTask {
-
-    private EventsFlattenerConfig config;
-    private JobMetrics metrics;
-    private EventsFlattenerService service;
-
-    public EventsFlattenerTask(Config config, TaskContext context) throws Exception {
-        init(config, context);
-    }
-
-    public EventsFlattenerTask() {
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) {
-        this.config = new EventsFlattenerConfig(config);
-        metrics = new JobMetrics(context, this.config.jobName());
-        service = new EventsFlattenerService(this.config);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator) {
-        EventsFlattenerSink sink = new EventsFlattenerSink(collector, metrics, config);
-        EventsFlattenerSource source = new EventsFlattenerSource(envelope);
-        service.process(source, sink);
-    }
-
-}
diff --git a/data-pipeline/share-events-flattener/src/main/resources/log4j.xml b/data-pipeline/share-events-flattener/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/share-events-flattener/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index e80f3a80f3..0000000000
--- a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,451 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-
-import java.util.Map;
-
-public class EventFixture {
-
-	public static final String VALID_SHARE_EVENT = "{\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"eid\": \"SHARE\",\n" +
-			"  \"ets\": 1577278681178,\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" +
-			"  },\n" +
-			"  \"context\": {\n" +
-			"    \"cdata\": [\n" +
-			"      {\n" +
-			"        \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" +
-			"        \"type\": \"API\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"id\": \"SearchResult\",\n" +
-			"        \"type\": \"Section\"\n" +
-			"      }\n" +
-			"    ],\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"prod.diksha.app\",\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"ver\": \"2.3.162\"\n" +
-			"    },\n" +
-			"    \"env\": \"app\",\n" +
-			"    \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" +
-			"    \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"dir\": \"In\",\n" +
-			"    \"type\": \"File\",\n" +
-			"    \"items\": [\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_312785709424099328114191\",\n" +
-			"        \"type\": \"CONTENT\",\n" +
-			"        \"ver\": \"1\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"transfers\": 0,\n" +
-			"            \"size\": 21084308\n" +
-			"          }\n" +
-			"        ]\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_31277435209002188818711\",\n" +
-			"        \"type\": \"CONTENT\",\n" +
-			"        \"ver\": \"18\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"transfers\": 12,\n" +
-			"            \"size\": \"123\"\n" +
-			"          }\n" +
-			"        ]\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_31278794857559654411554\",\n" +
-			"        \"type\": \"TextBook\",\n" +
-			"        \"ver\": \"1\"\n" +
-			"      }\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"do_312528116260749312248818\",\n" +
-			"    \"type\": \"TextBook\",\n" +
-			"    \"version\": \"10\",\n" +
-			"    \"rollup\": {}\n" +
-			"  },\n" +
-			"  \"mid\": \"02ba33e5-15fe-4ec5-b32.1084308E760-3d03429fae84\",\n" +
-			"  \"syncts\": 1577278682630,\n" +
-			"  \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-	public static final String VALID_SHARE_EVENT_WHEN_EMPTY_SIZE = "{\n" +
-			"  \"eid\": \"SHARE\",\n" +
-			"  \"ets\": 1578979395120,\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"mid\": \"SHARE:6a68490ac9c8ffcb8f8b8e1e6da1f09c\",\n" +
-			"  \"actor\": {\n" +
-			"    \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"    \"type\": \"User\"\n" +
-			"  },\n" +
-			"  \"context\": {\n" +
-			"    \"channel\": \"01231711180382208027\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"staging.diksha.desktop\",\n" +
-			"      \"ver\": \"1.0.3\",\n" +
-			"      \"pid\": \"desktop.app\"\n" +
-			"    },\n" +
-			"    \"env\": \"Content\",\n" +
-			"    \"sid\": \"d17137fd-224a-49bd-834c-0e9d7a21ca18\",\n" +
-			"    \"did\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"    \"cdata\": [],\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"01231711180382208027\"\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"object\": {},\n" +
-			"  \"tags\": [\n" +
-			"    \"01231711180382208027\"\n" +
-			"  ],\n" +
-			"  \"edata\": {\n" +
-			"    \"dir\": \"Out\",\n" +
-			"    \"type\": \"File\",\n" +
-			"    \"items\": [\n" +
-			"      {\n" +
-			"        \"type\": \"TextBook\",\n" +
-			"        \"ver\": \"1\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"transfers\": \"53\"\n" +
-			"          },\n" +
-			"          {\n" +
-			"            \"size\": \"\"\n" +
-			"          }\n" +
-			"        ],\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        }\n" +
-			"      }\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"syncts\": 1578979437523,\n" +
-			"  \"@timestamp\": \"2020-01-14T05:23:57.523Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-
-	public static final String VALID_SHARE_EVENT_WITHOU_SIZE = "{\n" +
-			"  \"eid\": \"SHARE\",\n" +
-			"  \"ets\": 1578979334609,\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"mid\": \"SHARE:336bd87993a20d2dc14a56d2ff197111\",\n" +
-			"  \"actor\": {\n" +
-			"    \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"    \"type\": \"User\"\n" +
-			"  },\n" +
-			"  \"context\": {\n" +
-			"    \"channel\": \"01231711180382208027\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"staging.diksha.desktop\",\n" +
-			"      \"ver\": \"1.0.3\",\n" +
-			"      \"pid\": \"desktop.app\"\n" +
-			"    },\n" +
-			"    \"env\": \"Content\",\n" +
-			"    \"sid\": \"d17137fd-224a-49bd-834c-0e9d7a21ca18\",\n" +
-			"    \"did\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"    \"cdata\": [],\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"01231711180382208027\"\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"object\": {},\n" +
-			"  \"tags\": [\n" +
-			"    \"01231711180382208027\"\n" +
-			"  ],\n" +
-			"  \"edata\": {\n" +
-			"    \"dir\": \"Out\",\n" +
-			"    \"type\": \"File\",\n" +
-			"    \"items\": [\n" +
-			"      {\n" +
-			"        \"type\": \"TextBook\",\n" +
-			"        \"ver\": \"2\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"transfers\": \"2\"\n" +
-			"          },\n" +
-			"          {\n" +
-			"            \"size\": \"66438\"\n" +
-			"          }\n" +
-			"        ],\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"373e0438854402794e7fc5c3e2feb5361b267fedcd6dfbd102b07a2882932e3a\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        }\n" +
-			"      }\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"syncts\": 1578979371914,\n" +
-			"  \"@timestamp\": \"2020-01-14T05:22:51.914Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-    public static final String VALID_START_EVENT = "{\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"eid\": \"START\",\n" +
-            "  \"ets\": 1577278681178,\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" +
-            "        \"type\": \"API\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"id\": \"SearchResult\",\n" +
-            "        \"type\": \"Section\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.app\",\n" +
-            "      \"pid\": \"sunbird.app\",\n" +
-            "      \"ver\": \"2.3.162\"\n" +
-            "    },\n" +
-            "    \"env\": \"app\",\n" +
-            "    \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" +
-            "    \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" +
-            "  },\n" +
-            "  \"edata\": {\n" +
-            "    \"dir\": \"In\",\n" +
-            "    \"type\": \"File\",\n" +
-            "    \"items\": [\n" +
-            "      {\n" +
-            "        \"origin\": {\n" +
-            "          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-            "          \"type\": \"Device\"\n" +
-            "        },\n" +
-            "        \"id\": \"do_312785709424099328114191\",\n" +
-            "        \"type\": \"CONTENT\",\n" +
-            "        \"ver\": \"1\",\n" +
-            "        \"params\": [\n" +
-            "          {\n" +
-            "            \"transfers\": 0,\n" +
-            "            \"size\": 21084308\n" +
-            "          }\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"origin\": {\n" +
-            "          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-            "          \"type\": \"Device\"\n" +
-            "        },\n" +
-            "        \"id\": \"do_31277435209002188818711\",\n" +
-            "        \"type\": \"CONTENT\",\n" +
-            "        \"ver\": \"18\",\n" +
-            "        \"params\": [\n" +
-            "          {\n" +
-            "            \"transfers\": 12,\n" +
-            "            \"size\": \"123\"\n" +
-            "          }\n" +
-            "        ]\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"origin\": {\n" +
-            "          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-            "          \"type\": \"Device\"\n" +
-            "        },\n" +
-            "        \"id\": \"do_31278794857559654411554\",\n" +
-            "        \"type\": \"TextBook\",\n" +
-            "        \"ver\": \"1\"\n" +
-            "      }\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"do_312528116260749312248818\",\n" +
-            "    \"type\": \"TextBook\",\n" +
-            "    \"version\": \"10\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" +
-            "  \"syncts\": 1577278682630,\n" +
-            "  \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-
-    public static final String INVALID_SHARE_EVENT = "{\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"eid\": \"SHARE\",\n" +
-			"  \"ets\": 1577278681178,\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" +
-			"  },\n" +
-			"  \"context\": {\n" +
-			"    \"cdata\": [\n" +
-			"      {\n" +
-			"        \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" +
-			"        \"type\": \"API\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"id\": \"SearchResult\",\n" +
-			"        \"type\": \"Section\"\n" +
-			"      }\n" +
-			"    ],\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"prod.diksha.app\",\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"ver\": \"2.3.162\"\n" +
-			"    },\n" +
-			"    \"env\": \"app\",\n" +
-			"    \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" +
-			"    \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"dir\": \"In\",\n" +
-			"    \"type\": \"File\",\n" +
-			"    \"items\": \"test\"\n" +
-			"  },\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"do_312528116260749312248818\",\n" +
-			"    \"type\": \"TextBook\",\n" +
-			"    \"version\": \"10\",\n" +
-			"    \"rollup\": {}\n" +
-			"  },\n" +
-			"  \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" +
-			"  \"syncts\": 1577278682630,\n" +
-			"  \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-    public static final String INVALID_EVENT = "{\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"eid\": \"SHARE\",\n" +
-			"  \"ets\": 1577278681178,\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"\n" +
-			"  },\n" +
-			"  \"context\": {\n" +
-			"    \"cdata\": [\n" +
-			"      {\n" +
-			"        \"id\": \"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\n" +
-			"        \"type\": \"API\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"id\": \"SearchResult\",\n" +
-			"        \"type\": \"Section\"\n" +
-			"      }\n" +
-			"    ],\n" +
-			"    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"id\": \"prod.diksha.app\",\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"ver\": \"2.3.162\"\n" +
-			"    },\n" +
-			"    \"env\": \"app\",\n" +
-			"    \"sid\": \"82e41d87-e33f-4269-aeae-d56394985599\",\n" +
-			"    \"did\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"dir\": \"In\",\n" +
-			"    \"type\": \"File\",\n" +
-			"    \"items\": [\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_312785709424099328114191\",\n" +
-			"        \"type\": \"CONTENT\",\n" +
-			"        \"ver\": \"1\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"size\": 21084308\n" +
-			"          }\n" +
-			"        ]\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_31277435209002188818711\",\n" +
-			"        \"type\": \"CONTENT\",\n" +
-			"        \"ver\": \"18\",\n" +
-			"        \"params\": [\n" +
-			"          {\n" +
-			"            \"transfers\": 12,\n" +
-			"            \"size\": \"123\"\n" +
-			"          }\n" +
-			"        ]\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"origin\": {\n" +
-			"          \"id\": \"1b17c32bad61eb9e33df281eecc727590d739b2b\",\n" +
-			"          \"type\": \"Device\"\n" +
-			"        },\n" +
-			"        \"id\": \"do_31278794857559654411554\",\n" +
-			"        \"type\": \"TextBook\",\n" +
-			"        \"ver\": \"1\"\n" +
-			"      }\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"do_312528116260749312248818\",\n" +
-			"    \"type\": \"TextBook\",\n" +
-			"    \"version\": \"10\",\n" +
-			"    \"rollup\": {}\n" +
-			"  },\n" +
-			"  \"mid\": \"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\n" +
-			"  \"syncts\": 1577278682630,\n" +
-			"  \"@timestamp\": \"2019-12-25T12:58:02.630Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true,\n" +
-			"    \"dd_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-	public static Map<String, Object> getMap(String message) {
-		return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-	}
-	
-}
diff --git a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java b/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java
deleted file mode 100644
index 6fdd6d8ae4..0000000000
--- a/data-pipeline/share-events-flattener/src/test/java/org/ekstep/ep/samza/task/EventsFlattenerTaskTest.java
+++ /dev/null
@@ -1,165 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Matchers;
-import org.mockito.Mockito;
-
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertNull;
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Mockito.*;
-
-public class EventsFlattenerTaskTest {
-
-    private static final String TELEMETRY_EVENTS_TOPIC = "telemetry.sink";
-    private static final String FAILED_TOPIC = "telemetry.failed";
-
-    private MessageCollector collectorMock;
-    private TaskContext contextMock;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private Config configMock;
-    private EventsFlattenerTask eventsFlattenerTask;
-
-    @Before
-    public void setUp() {
-        collectorMock = mock(MessageCollector.class);
-        contextMock = Mockito.mock(TaskContext.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        counter = Mockito.mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        configMock = Mockito.mock(Config.class);
-
-        stub(configMock.get("output.success.topic.name", TELEMETRY_EVENTS_TOPIC)).toReturn(TELEMETRY_EVENTS_TOPIC);
-        stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-    }
-
-    /**
-     * When share event having multiple edata items
-     */
-    @Test
-    public void shouldFlattenTheShareEvent() {
-        try {
-            eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock);
-            stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WITHOU_SIZE);
-            eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock);
-            Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true)));
-        } catch (Exception e) {
-           System.out.println(e.getMessage());
-        }
-    }
-
-    @Test
-    public void shouldFlattenTheShareEventWhenParamsHavingMultipleObject() {
-        try {
-            eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock);
-            stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT);
-            eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock);
-            Mockito.verify(collectorMock, times(4)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true)));
-        } catch (Exception e) {
-            e.printStackTrace();
-            System.out.println(e.getMessage());
-        }
-    }
-
-    @Test
-    public void shouldFlattenTheEventWhenTransferIsNul() {
-        try {
-            eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock);
-            stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WITHOU_SIZE);
-            eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock);
-            Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true)));
-        } catch (Exception e) {
-            System.out.println(e.getMessage());
-        }
-    }
-
-    @Test
-    public void shouldFlattenTheEventWhenSizeIsEmpty() {
-        try {
-            eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock);
-            stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_SHARE_EVENT_WHEN_EMPTY_SIZE);
-            eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock);
-            Mockito.verify(collectorMock, times(2)).send(Matchers.argThat(validateEventObject(Arrays.asList("File", "import", "download"), true)));
-        } catch (Exception e) {
-            System.out.println(e.getMessage());
-        }
-    }
-
-    /**
-     * When Event is other than "SHARE" Then it should route to success topic
-     */
-    @Test
-    public void shouldRouteToSuccessTopic() {
-        try {
-            stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_START_EVENT);
-            eventsFlattenerTask = new EventsFlattenerTask(configMock, contextMock);
-            eventsFlattenerTask.process(envelopeMock, collectorMock, coordinatorMock);
-            verify(collectorMock, times(4)).send(argThat(validateOutputTopic(envelopeMock.getMessage(), TELEMETRY_EVENTS_TOPIC)));
-        } catch (Exception e) {
-            System.out.println(e.getMessage());
-        }
-    }
-
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertNotNull(outgoingMessageEnvelope.getMessage());
-                return true;
-            }
-        };
-    }
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateEventObject(List<String> edataType, Boolean ef_processed) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                Map<String, Object> shareEvent;
-                Map<String, Object> shareEventEdata;
-                OutgoingMessageEnvelope flattenEvent = (OutgoingMessageEnvelope) o;
-                String message = (String) flattenEvent.getMessage();
-                System.out.println(message);
-                shareEvent = new Gson().fromJson(message, Map.class);
-                assertEquals(ef_processed, new Gson().fromJson(shareEvent.get("flags").toString(), Map.class).get("ef_processed"));
-                if (shareEvent.get("eid").equals("SHARE_ITEM")) {
-                    shareEventEdata = new Gson().fromJson(new Gson().toJson(shareEvent.get("edata")), Map.class);
-                    assertEquals(true, edataType.contains(shareEventEdata.get("type")));
-                    assertNull(shareEventEdata.get("items"));
-                } else {
-                    shareEventEdata = new Gson().fromJson(new Gson().toJson(shareEvent.get("edata")), Map.class);
-                    assertNotNull(shareEventEdata.get("items"));
-                }
-                return true;
-            }
-        };
-    }
-}
diff --git a/data-pipeline/telemetry-extractor/pom.xml b/data-pipeline/telemetry-extractor/pom.xml
deleted file mode 100644
index a5a7a6e83e..0000000000
--- a/data-pipeline/telemetry-extractor/pom.xml
+++ /dev/null
@@ -1,175 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>telemetry-extractor</artifactId>
-    <version>0.1.5</version>
-    <packaging>jar</packaging>
-    <name>TelemetryExtractor</name>
-    <description>
-        Stamp @timestmap in telemetry
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.scala-lang/scala-reflect -->
-        <dependency>
-            <groupId>org.scala-lang</groupId>
-            <artifactId>scala-reflect</artifactId>
-            <version>2.11.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.8.0</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/telemetry-extractor/src/main/assembly/src.xml b/data-pipeline/telemetry-extractor/src/main/assembly/src.xml
deleted file mode 100644
index 6e0aa8d169..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/assembly/src.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/telemetry-extractor.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:telemetry-extractor</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties b/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties
deleted file mode 100644
index 03b4ca2a0e..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/config/telemetry-extractor.properties
+++ /dev/null
@@ -1,116 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-####Environment
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.TelemetryExtractor
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_extractor_yarn_container_count__
-yarn.container.memory.mb=__telemetry_extractor_container_memory_mb__
-
-# Task
-task.class=org.ekstep.ep.samza.task.TelemetryExtractorTask
-task.inputs=kafka.__env__.telemetry.ingest
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-# Normally, this would be 3, but we have only one broker.
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot,jmx
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-metrics.reporter.jmx.class=org.apache.samza.metrics.reporter.JmxReporterFactory
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.msg.serde=string
-systems.kafka.samza.key.serde=string
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.consumer.fetch.message.max.bytes=__telemetry_extractor_consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__telemetry_extractor_consumer_fetch_max_bytes__
-systems.kafka.samza.fetch.threshold=__telemetry_extractor_messages_fetch_threshold__
-
-
-## Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.success.topic.name=__env__.telemetry.raw
-output.assess.topic.name=__env__.telemetry.assess.redact
-output.error.topic.name=__env__.telemetry.extractor.failed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count,batch-success-count,batch-error-count,duplicate-event-count,assess-route-success-count
-default.channel=__default_channel__
-
-# redis
-#redis.host=localhost
-redis.host=__redis_host__
-#redis.port=6379
-redis.port=__redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.duplicationstore.id=1
-redis.database.key.expiry.seconds=432000
-raw.individual.event.maxsize=996148
-
-####Local
-## Job
-#job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-#job.name=TelemetryExtractor
-
-## YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-
-## Task
-#task.inputs=kafka.telemetry.ingest
-
-## Metrics
-#metrics.reporter.snapshot.stream=kafka.metrics
-
-##Key-Value store
-#stores.telemetry-extractor.changelog=kafka.telemetry-extractor-changelog
-
-## Systems
-#systems.kafka.consumer.zookeeper.connect=localhost:2181/
-#systems.kafka.producer.bootstrap.servers=localhost:9092
-
-## Job Coordinator
-#output.success.topic.name=telemetry.raw
-#output.error.topic.name=telemetry.extractor.failed
-#output.metrics.topic.name=pipeline_metrics
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java
deleted file mode 100755
index 5aacd0308b..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Actor.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import java.util.Map;
-
-public class Actor {
-    private String id = "";
-    private String type = "telemetry-sync";
-
-    public Actor() {
-
-    }
-
-    public Actor(Map<String, Object> eventSpec) {
-        this.id = (String) eventSpec.get("id");
-    }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java
deleted file mode 100755
index 85d6e0def1..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/CData.java
+++ /dev/null
@@ -1,26 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import java.util.Map;
-
-public class CData {
-    private String type;
-    private String id;
-
-    public CData(Map<String, Object> cdata) {
-        this.id = (String) cdata.get("id");
-        this.type = (String) cdata.get("type");
-    }
-
-    public CData(String type, String id) {
-        this.type = type;
-        this.id = id;
-    }
-
-    public String getType() {
-        return type;
-    }
-
-    public String getId() {
-        return id;
-    }
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java
deleted file mode 100755
index 3a65b488e2..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Context.java
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.annotations.SerializedName;
-
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.task.TelemetryExtractorConfig;
-
-import java.util.*;
-
-public class Context {
-    private String channel = "";
-    private String env = "";
-    private String sid = "";
-    private String did = "";
-    static Logger LOGGER = new Logger(Context.class);
-
-    @SerializedName("pdata")
-    private Map<String, String> pData = new HashMap<String, String>();
-
-    @SerializedName("cdata")
-    private ArrayList<CData> cData = new ArrayList<>();
-
-
-    public Context(Map<String, Object> eventSpec, String defaultChannel) {
-
-        this.channel = defaultChannel;
-        this.pData.put("id", "pipeline");
-        this.pData.put("pid", "");
-        this.pData.put("ver", "");
-
-        try {
-            List<Map<String, Object>> events = (List<Map<String, Object>>) eventSpec.get("events");
-            Map<String, Object> event = events.get(0);
-            Map<String, Object> eventContext = (Map<String, Object>) event.get("context");
-            env = "telemetry-sync";
-            did = (String) eventContext.get("did");
-            String channel = (String) eventContext.get("channel");
-            if (channel != null && !"".equals(channel.trim())) {
-                this.channel = channel;
-            }
-            Map<String, String> pdata = (Map<String, String>) eventContext.get("pdata");
-            if (pdata != null && pdata.containsKey("id")) {
-                this.pData = pdata;
-            }
-
-        } catch (Exception e) {
-            LOGGER.info("", "Failed to initialize context: " + e.getMessage());
-        }
-
-    }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java
deleted file mode 100755
index af26bbafa1..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/TObject.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.annotations.SerializedName;
-
-import java.util.Map;
-
-public class TObject {
-    private String id;
-    private String type;
-    private String ver;
-
-    @SerializedName("subtype")
-    private String subType;
-
-    public TObject(Map<String, Object> eventSpec) {
-
-        this.id = (String) eventSpec.get("mid");
-        this.type = "event";
-
-    }
-
-    public String getId() {
-        return id;
-    }
-
-    public void setId(String id) {
-        this.id = id;
-    }
-
-    public String getType() {
-        return type;
-    }
-
-    public void setType(String type) {
-        this.type = type;
-    }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java
deleted file mode 100755
index e04d9a0709..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/domain/Telemetry.java
+++ /dev/null
@@ -1,123 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import org.ekstep.ep.samza.core.Logger;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class Telemetry {
-
-    static Logger LOGGER = new Logger(Telemetry.class);
-
-    private final String ver = "3.0";
-    private String eid;
-    private long ets;
-    private String mid;
-    private Actor actor;
-    private Context context;
-    private TObject object;
-    private HashMap<String, Object> edata;
-    private List<String> tags = new ArrayList<>();
-    private HashMap<String, String> metadata;
-    private long syncts;
-    private String syncTimestamp;
-
-    public Telemetry() {
-
-    }
-
-    public Telemetry(Map<String, Object> batchEvent, long syncts, String syncTimestamp, String defaultChannel) {
-
-        try {
-            @SuppressWarnings("unchecked")
-            List<Map<String, Object>> events = (List<Map<String, Object>>) batchEvent.get("events");
-
-            // TODO - Handle NPE if the batchEvent doesn't have 'ets'. Default 'ets' to System.currentTimeInMillis()
-            this.ets = ((Number) batchEvent.get("ets")).longValue();
-            this.syncts = syncts;
-            this.syncTimestamp = syncTimestamp;
-
-            String mid = (String) batchEvent.get("mid");
-
-            int eventCount = events.size();
-            String status = (String) batchEvent.get("sync_status");
-            String ver = (String) batchEvent.get("ver");
-            if (null == status) {
-                status = "SUCCESS";
-            }
-            String consumerId = (String) batchEvent.get("consumer_id");
-            if (null == consumerId) {
-                consumerId = "";
-            }
-
-            HashMap<String, Object> edata = new HashMap<String, Object>();
-            edata.put("type", "telemetry_audit");
-            edata.put("level", "INFO");
-            edata.put("message", "telemetry sync");
-            edata.put("pageid", "data-pipeline");
-
-            List<Map<String, Object>> params = new ArrayList<Map<String, Object>>();
-            Map<String, Object> param = new HashMap<>();
-            param.put("sync_status", status);
-            param.put("consumer_id", consumerId);
-            param.put("events_count", eventCount);
-            param.put("ver", ver);
-            params.add(param);
-            edata.put("params", params);
-
-            this.eid = "LOG";
-            this.edata = edata;
-            this.mid = computeMid(this.eid, mid);
-            this.metadata = new HashMap<>();
-            this.actor = new Actor(batchEvent);
-            this.context = new Context(batchEvent, defaultChannel);
-            this.object = new TObject(batchEvent);
-        } catch (Exception e) {
-
-            LOGGER.info("", "Failed to initialize telemetry spec data: " + e.getMessage());
-        }
-
-    }
-
-    public String computeMid(String eid, String mid) {
-        // Because v2->v3 is one to many, mids have to be changed
-        // We just prefix the LOG EID with telemetry spec mid
-        return String.format("%s:%s", eid, mid);
-    }
-
-
-    public TObject getObject() {
-        return object;
-    }
-
-    public void setObject(TObject object) {
-        this.object = object;
-    }
-
-
-    public Map<String, Object> toMap() {
-        Map<String, Object> v3map = new HashMap<>();
-        v3map.put("eid", this.eid);
-        v3map.put("ets", this.ets);
-        v3map.put("ver", this.ver);
-        v3map.put("mid", this.mid);
-        v3map.put("actor", this.actor);
-        v3map.put("context", this.context);
-        v3map.put("object", this.object);
-        v3map.put("metadata", this.metadata);
-        v3map.put("edata", this.edata);
-        v3map.put("tags", this.tags);
-        v3map.put("syncts", this.syncts);
-        v3map.put("@timestamp", this.syncTimestamp);
-        return v3map;
-    }
-
-    public String toJson() {
-        Map<String, Object> map = toMap();
-        return new Gson().toJson(map);
-    }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java
deleted file mode 100644
index 4dffe171d1..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/service/TelemetryExtractorService.java
+++ /dev/null
@@ -1,149 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.Gson;
-
-import redis.clients.jedis.exceptions.JedisException;
-
-import org.apache.commons.lang.StringUtils;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Telemetry;
-import org.ekstep.ep.samza.task.TelemetryExtractorConfig;
-import org.ekstep.ep.samza.task.TelemetryExtractorSink;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class TelemetryExtractorService {
-
-  static Logger LOGGER = new Logger(TelemetryExtractorService.class);
-
-  private DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC();
-  private JobMetrics metrics;
-  private DeDupEngine deDupEngine;
-  private String defaultChannel = "";
-  private int rawIndividualEventMaxSize;
-  private List<String> assessEvents = Arrays.asList("ASSESS", "RESPONSE");
-
-  public TelemetryExtractorService(TelemetryExtractorConfig config, JobMetrics metrics, DeDupEngine deDupEngine) {
-    this.metrics = metrics;
-    this.deDupEngine = deDupEngine;
-    this.defaultChannel = config.defaultChannel();
-    this.rawIndividualEventMaxSize = config.rawIndividualEventMaxSize();
-  }
-
-  @SuppressWarnings("unchecked")
-  public void process(String message, TelemetryExtractorSink sink) {
-
-    try {
-      Map<String, Object> batchEvent = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-      long syncts = getSyncTS(batchEvent);
-      String syncTimestamp = df.print(syncts);
-
-      if (batchEvent.containsKey("params") && null != batchEvent.get("params")) {
-        String msgid = "";
-        Map<String, Object> params = (Map<String, Object>) batchEvent.get("params");
-        if (params.containsKey("msgid") && null != params.get("msgid")) {
-          msgid = params.get("msgid").toString();
-          if (!deDupEngine.isUniqueEvent(msgid)) {
-            LOGGER.info("", String.format("msgid: %s: DUPLICATE EVENT", msgid));
-            sink.toDuplicateTopic(addDuplicateFlag(batchEvent));
-            return;
-          }
-          deDupEngine.storeChecksum(msgid);
-        }
-
-      }
-
-      List<Map<String, Object>> events = (List<Map<String, Object>>) batchEvent.get("events");
-
-      for (Map<String, Object> event : events) {
-        String json = "";
-        try {
-          event.put("syncts", syncts);
-          event.put("@timestamp", syncTimestamp);
-          Map<String, Object> context = (Map<String, Object>) event.get("context");
-          String channel = (String) context.get("channel");
-          String eid = (String) event.get("eid");
-          if (StringUtils.isEmpty(channel)) {
-            event.put("context", context);
-          }
-          json = new Gson().toJson(event);
-          int eventSizeInBytes = json.getBytes("UTF-8").length;
-          if (eventSizeInBytes > rawIndividualEventMaxSize) {
-            LOGGER.info("",
-                String.format("Event with mid %s of size %d bytes is greater than %d. " + "Sending to error topic",
-                    event.get("mid"), eventSizeInBytes, rawIndividualEventMaxSize));
-            sink.toErrorTopic(json);
-          } else {
-            if (assessEvents.contains(eid)) {
-              sink.toAssessTopic(json);
-            } else {
-              sink.toSuccessTopic(json);
-            }
-          }
-        } catch (Throwable t) {
-          LOGGER.info("", "Failed to send extracted event to success topic: " + t.getMessage());
-          sink.toErrorTopic(json);
-        }
-      }
-      metrics.incBatchSuccessCounter();
-      generateAuditEvent(batchEvent, syncts, syncTimestamp, sink, defaultChannel);
-    } catch (JedisException e) {
-      e.printStackTrace();
-      LOGGER.error(null, "Exception when retrieving data from redis: ", e);
-      deDupEngine.getRedisConnection().close();
-      throw e;
-    } catch (Exception ex) {
-      ex.printStackTrace();
-      LOGGER.error("", "Failed to extract the event batch: ", ex);
-      sink.sinkBatchErrorEvents(message);
-    }
-
-  }
-
-  private long getSyncTS(Map<String, Object> batchEvent) {
-
-    if (batchEvent.containsKey("syncts")) {
-      Object obj = batchEvent.get("syncts");
-      if (obj instanceof Number) {
-        return ((Number) obj).longValue();
-      }
-    }
-
-    return System.currentTimeMillis();
-  }
-
-  /**
-   * Create LOG event to audit telemetry sync
-   *
-   * @param eventSpec
-   * @param syncts
-   * @param syncTimestamp
-   * @param sink
-   */
-  private void generateAuditEvent(Map<String, Object> eventSpec, long syncts, String syncTimestamp,
-      TelemetryExtractorSink sink, String defaultChannel) {
-    try {
-      Telemetry v3spec = new Telemetry(eventSpec, syncts, syncTimestamp, defaultChannel);
-      String auditEvent = v3spec.toJson();
-      sink.toSuccessTopic(auditEvent);
-    } catch (Exception e) {
-      e.printStackTrace();
-      LOGGER.debug("", "Failed to generate LOG event: " + e.getMessage());
-    }
-  }
-
-  public String addDuplicateFlag(Map<String, Object> batchEvent) {
-    Map<String, String> flags = new HashMap<>();
-    flags.put("extractor_duplicate", "true");
-    batchEvent.put("flags", flags);
-    return new Gson().toJson(batchEvent);
-  }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java
deleted file mode 100644
index 78f413b0d6..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorConfig.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-public class TelemetryExtractorConfig {
-	
-	private final String JOB_NAME = "TelemetryExtractor";
-	private final String errorTopic;
-	private String successTopic;
-	private String assessTopic;
-	private String defaultChannel;
-	private final int dupStore;
-	private int expirySeconds;
-	private int rawIndividualEventMaxSize;
-
-	public TelemetryExtractorConfig(Config config) {
-		successTopic = config.get("output.success.topic.name", "telemetry.raw");
-		assessTopic = config.get("output.assess.topic.name", "telemetry.assess.redact");
-		errorTopic = config.get("output.error.topic.name", "telemetry.extractor.failed");
-		defaultChannel = config.get("default.channel", "01250894314817126443");
-		dupStore = config.getInt("redis.database.duplicationstore.id", 1);
-		expirySeconds = config.getInt("redis.database.key.expiry.seconds", 432000);
-		rawIndividualEventMaxSize = config.getInt("raw.individual.event.maxsize", 996148);
-	}
-
-	public String successTopic() {
-		return successTopic;
-	}
-	
-	public String assessTopic() {
-    return assessTopic;
-  }
-
-	public String errorTopic() {
-		return errorTopic;
-	}
-
-	public String jobName() {
-		return JOB_NAME;
-	}
-
-	public String defaultChannel() {
-		return defaultChannel;
-	}
-
-	public int dupStore() {
-		return dupStore;
-	}
-
-	public int expirySeconds() {
-		return expirySeconds;
-	}
-
-	public int rawIndividualEventMaxSize() {
-		return rawIndividualEventMaxSize;
-	}
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java
deleted file mode 100644
index 40d9be5f8b..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorSink.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-public class TelemetryExtractorSink extends BaseSink {
-
-	private TelemetryExtractorConfig config;
-
-	public TelemetryExtractorSink(MessageCollector collector, JobMetrics metrics, TelemetryExtractorConfig config) {
-		super(collector, metrics);
-		this.config = config;
-	}
-	
-	public void toSuccessTopic(String message) {
-		metrics.incSuccessCounter();
-		toTopic(config.successTopic(), null, message);
-	}
-	
-	public void toAssessTopic(String message) {
-    metrics.incAssessRouteSuccessCounter();
-    toTopic(config.assessTopic(), null, message);
-  }
-	
-	public void toErrorTopic(String message) {
-		metrics.incErrorCounter();
-		toTopic(config.errorTopic(), null, message);
-	}
-
-	public void sinkBatchErrorEvents(String message) {
-		metrics.incBatchErrorCounter();
-		toTopic(config.errorTopic(), null, message);
-	}
-
-	public void toDuplicateTopic(String message)
-	{
-		metrics.incDuplicateCounter();
-		toTopic(config.errorTopic(), null, message);
-	}
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java b/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java
deleted file mode 100644
index 3b35945eb2..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/java/org/ekstep/ep/samza/task/TelemetryExtractorTask.java
+++ /dev/null
@@ -1,70 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.TelemetryExtractorService;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class TelemetryExtractorTask extends BaseSamzaTask {
-
-	private TelemetryExtractorConfig config;
-	private TelemetryExtractorService service;
-	private JobMetrics metrics;
-
-	public TelemetryExtractorTask(Config config, TaskContext context, DeDupEngine deDupEngine) {
-		init(config, context, deDupEngine);
-	}
-
-	public TelemetryExtractorTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		init(config, context, null);
-	}
-
-
-	public void init(Config config, TaskContext context, DeDupEngine deDupEngine) {
-		this.config = new TelemetryExtractorConfig(config);
-		this.metrics = new JobMetrics(context, this.config.jobName());
-		deDupEngine = deDupEngine == null ?
-				new DeDupEngine(new RedisConnect(config), this.config.dupStore(), this.config.expirySeconds()) : deDupEngine;
-		this.service = new TelemetryExtractorService(this.config, this.metrics,deDupEngine);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator)
-			throws Exception {
-
-		String message = (String) envelope.getMessage();
-		TelemetryExtractorSink sink = new TelemetryExtractorSink(collector, metrics, config);
-		service.process(message, sink);
-	}
-
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml b/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/telemetry-extractor/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java
deleted file mode 100644
index 2c832f8006..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/domain/TestCdata.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.junit.Assert.assertTrue;
-
-public class TestCdata {
-
-    @Test
-    public void shouldIntializeTheCdataValues() {
-        CData cData = new CData("contentSession", "c803439b26d433fa920200067ebe4408");
-        assertTrue(cData.getType() == "contentSession");
-        assertTrue(cData.getId() == "c803439b26d433fa920200067ebe4408");
-
-    }
-
-    @Test
-    public void shouldGetTheCdataValues() {
-        Map<String, Object> cdata = new HashMap<>();
-        cdata.put("type", "contentSession");
-        cdata.put("id", "c803439b26d433fa920200067ebe4408");
-
-        CData cData = new CData(cdata);
-        assertTrue(cData.getType() == "contentSession");
-        assertTrue(cData.getId() == "c803439b26d433fa920200067ebe4408");
-
-    }
-
-}
diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index c517273eb0..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.util.Map;
-
-public class EventFixture {
-    public static Map<String, Object> getEvent(String eventId) throws FileNotFoundException {
-        URL url = Thread.currentThread().getContextClassLoader().getResource(eventId + ".json");
-        File file = new File(url.getPath());
-        Type type = new TypeToken<Map<String, Object>>(){}.getType();
-        return new Gson().fromJson(new FileReader(file), type);
-    }
-
-    public static String getEventAsString(String eventId) throws IOException, URISyntaxException {
-        URL url = Thread.currentThread().getContextClassLoader().getResource(eventId + ".json");
-        return new String(Files.readAllBytes(Paths.get(url.toURI())));
-    }
-}
diff --git a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java b/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java
deleted file mode 100644
index 333328e25a..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/java/org/ekstep/ep/samza/task/TelemetryExtractorTaskTest.java
+++ /dev/null
@@ -1,264 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import static junit.framework.Assert.assertNotNull;
-import static junit.framework.Assert.assertNull;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-import java.lang.reflect.Type;
-import java.util.List;
-import java.util.Map;
-
-import com.google.gson.reflect.TypeToken;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.DeDupEngine;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-
-import com.google.gson.Gson;
-import redis.clients.jedis.Jedis;
-
-public class TelemetryExtractorTaskTest {
-
-    class TestMessageCollector implements MessageCollector {
-
-        public OutgoingMessageEnvelope outgoingEnvelope;
-
-        @Override
-        public void send(OutgoingMessageEnvelope outgoingMessageEnvelope) {
-            outgoingEnvelope = outgoingMessageEnvelope;
-        }
-    }
-
-    private TaskContext context;
-    private Config config;
-    private IncomingMessageEnvelope envelope;
-    private MessageCollector collector;
-    private TaskCoordinator coordinator;
-    private Counter counter;
-    private MetricsRegistry metricsRegistry;
-    private final String successTopic = "telemetry.raw";
-    private final String errorTopic = "telemetry.extractor.failed";
-    private final String defaultChannel = "01250894314817126443";
-    private Type mapType = new TypeToken<Map<String, Object>>() {
-    }.getType();
-    TelemetryExtractorTask task;
-    private DeDupEngine deDupEngineMock;
-
-    @Before
-    public void setup() {
-
-        context = mock(TaskContext.class);
-        config = mock(Config.class);
-        envelope = mock(IncomingMessageEnvelope.class);
-        collector = new TestMessageCollector();
-        coordinator = mock(TaskCoordinator.class);
-        counter = mock(Counter.class);
-        metricsRegistry = mock(MetricsRegistry.class);
-        stub(context.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(metricsRegistry.newCounter(anyString(), anyString()))
-                .toReturn(counter);
-        stub(envelope.getOffset()).toReturn("2");
-        stub(envelope.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1)));
-        stub(config.get("output.success.topic.name", "telemetry.raw")).toReturn(successTopic);
-        stub(config.get("output.assess.topic.name", "telemetry.assess.redact")).toReturn("telemetry.assess.redact");
-        stub(config.get("output.error.topic.name", "telemetry.extractor.failed")).toReturn(errorTopic);
-        stub(config.get("default.channel", "01250894314817126443")).toReturn(defaultChannel);
-        stub(config.get("default.channel", "01250894314817126443")).toReturn(defaultChannel);
-        stub(config.getInt("raw.individual.event.maxsize", 996148)).toReturn(996148);
-        deDupEngineMock = mock(DeDupEngine.class);
-            task = new TelemetryExtractorTask(config, context, deDupEngineMock);
-
-    }
-    
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateEventObject() {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope envelope = (OutgoingMessageEnvelope) o;
-                String message = (String) envelope.getMessage();
-                SystemStream stream = envelope.getSystemStream();
-                Map<String, Object> event = new Gson().fromJson(message, Map.class);
-                System.out.println("Eid" + event.get("eid"));
-                
-                if ("ASSESS".equals(event.get("eid"))) {
-                    assertEquals("ASSESS:6ac822896cd8a1736d55806c13ada64c", event.get("mid"));
-                    assertEquals("kafka", stream.getSystem());
-                    assertEquals("telemetry.assess.redact", stream.getStream());
-                } else if ("RESPONSE".equals(event.get("eid"))) {
-                    assertEquals("RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef", event.get("mid"));
-                    assertEquals("kafka", stream.getSystem());
-                    assertEquals("telemetry.assess.redact", stream.getStream());
-                } else {
-                    assertEquals("kafka", stream.getSystem());
-                    assertEquals("telemetry.raw", stream.getStream());
-                }
-                return true;
-            }
-        };
-    }
-
-    @Test
-    public void synctsShouldBeStamped() throws Exception {
-        String spec = EventFixture.getEventAsString("event");
-        stub(envelope.getMessage()).toReturn(spec);
-
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.raw", stream.getStream());
-
-        Map<String, Object> output = new Gson().fromJson((String) envelope.getMessage(), mapType);
-        assertTrue(output.containsKey("syncts"));
-        assertTrue(output.containsKey("@timestamp"));
-        assertEquals(1529500243955L, ((Number) output.get("syncts")).longValue());
-        assertEquals("2018-06-20T13:10:43.955Z", output.get("@timestamp"));
-
-    }
-
-    @Test
-    public void assessEventShouldBeRouted() throws Exception {
-        String spec = EventFixture.getEventAsString("event");
-        stub(envelope.getMessage()).toReturn(spec);
-        MessageCollector collectorMock = mock(MessageCollector.class);
-        
-        task.process(envelope, collectorMock, coordinator);
-        verify(collectorMock, times(23)).send(argThat(validateEventObject()));
-    }
-    
-    @Test
-    public void shouldSendDuplicateBatchEventToFailedTopic() throws Exception {
-        String spec = EventFixture.getEventAsString("event_app");
-        stub(envelope.getMessage()).toReturn(spec);
-        stub(deDupEngineMock.isUniqueEvent(anyString())).toReturn(false);
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.extractor.failed", stream.getStream());
-
-        Map<String, Object> output = new Gson().fromJson((String) envelope.getMessage(), mapType);
-        assertTrue(output.containsKey("flags"));
-
-    }
-
-
-    @Test
-    public void eventCountShouldZero() throws Exception {
-        String spec = EventFixture.getEventAsString("empty_events");
-        stub(envelope.getMessage()).toReturn(spec);
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.raw", stream.getStream());
-
-        String output = (String) envelope.getMessage();
-        Map<String, Object> event = (Map<String, Object>) new Gson().fromJson(output, Map.class);
-        List<Map<String, Object>> edata = (List<Map<String, Object>>) ((Map<String, Object>) event.get("edata")).get("params");
-        int event_count = ((Number) edata.get(0).get("events_count")).intValue();
-        assertEquals(0, event_count);
-
-        Map<String, Object> context = (Map<String, Object>) event.get("context");
-        String channel = (String) context.get("channel");
-        assertEquals("01250894314817126443", channel);
-
-        Map<String, String> padata = (Map<String, String>) context.get("pdata");
-        assertEquals("pipeline", padata.get("id"));
-    }
-
-    @Test
-    public void specWithoutSyncts() throws Exception {
-        String spec = EventFixture.getEventAsString("event_without_syncts");
-        stub(envelope.getMessage()).toReturn(spec);
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        String output = (String) envelope.getMessage();
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.raw", stream.getStream());
-        assertEquals(true, output.contains("syncts"));
-    }
-
-    @Test
-    public void specDoesNotContainJsonData() throws Exception {
-        String spec = EventFixture.getEventAsString("not_json_event");
-        stub(envelope.getMessage()).toReturn(spec);
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        String output = (String) envelope.getMessage();
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.extractor.failed", stream.getStream());
-        List<String> events = (List<String>) ((Map<String, Object>) new Gson().fromJson(output, Map.class)).get("events");
-        String content = events.get(0);
-        assertEquals("testing events", content);
-    }
-
-    @Test
-    public void specDoesNotContainEventsKey() throws Exception {
-        String spec = EventFixture.getEventAsString("without_event_key");
-        stub(envelope.getMessage()).toReturn(spec);
-
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        String output = (String) envelope.getMessage();
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("telemetry.extractor.failed", stream.getStream());
-        assertEquals(false, output.contains("events"));
-    }
-
-    @Test
-    public void auditEventMetrics() throws Exception {
-        String spec = EventFixture.getEventAsString("event1");
-        stub(envelope.getMessage()).toReturn(spec);
-
-        task.process(envelope, collector, coordinator);
-
-        OutgoingMessageEnvelope envelope = ((TestMessageCollector) collector).outgoingEnvelope;
-        SystemStream stream = envelope.getSystemStream();
-
-        String output = (String) envelope.getMessage();
-        Map<String, Object> event = (Map<String, Object>) new Gson().fromJson(output, Map.class);
-        assertEquals("kafka", stream.getSystem());
-        assertEquals("LOG", (String) event.get("eid"));
-        Map<String, Object> edata = (Map<String, Object>) event.get("edata");
-        assertEquals("telemetry_audit", (String) edata.get("type"));
-        assertEquals("INFO", (String) edata.get("level"));
-
-        Map<String, Object> param = (Map<String, Object>) ((List<Object>) edata.get("params")).get(0);
-        assertEquals(2, ((Number) param.get("events_count")).intValue());
-        assertEquals("SUCCESS", (String) param.get("sync_status"));
-    }
-}
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json b/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json
deleted file mode 100644
index 0cd7a82dc6..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/empty_events.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event.json b/data-pipeline/telemetry-extractor/src/test/resources/event.json
deleted file mode 100644
index 996187ff97..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/event.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"RESPONSE","ets":1586759864200,"ver":"3.0","mid":"RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef","actor":{"id":"c5b5732b-b8d6-44ad-bf52-d8f7fc513ac3","type":"User"},"context":{"channel":"0126825293972439041","pdata":{"id":"preprod.diksha.portal","ver":"2.8.7","pid":"sunbird-portal.contentplayer"},"env":"contentplayer","sid":"zSizgfO7mzdZ--MhlGCzszgdNhKW6MCp","did":"a915339fbca0afddbf537a1792f01791","cdata":[{"id":"28d1f55313c9c14df8ea86ca1dc73a92","type":"ContentSession"},{"id":"fed7932935f2223870beb120bf58aeed","type":"PlaySession"}],"rollup":{"l1":"0126825293972439041"}},"object":{"id":"do_2129946748092334081914","type":"Content","ver":"2","rollup":{}},"tags":["0126825293972439041"],"edata":{"target":{"id":"do_21299463432309145612386","ver":"1.0","type":"AssessmentItem"},"type":"CHOOSE","values":[{"option0":"<p>1</p>\n"}]}},{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"ASSESS","ets":1529499987116,"ver":"3.0","mid":"ASSESS:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event1.json b/data-pipeline/telemetry-extractor/src/test/resources/event1.json
deleted file mode 100644
index 73e606e079..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/event1.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event_app.json b/data-pipeline/telemetry-extractor/src/test/resources/event_app.json
deleted file mode 100644
index e194e7fe18..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/event_app.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"params":{"msgid":"3fc11963-04e7-4251-83de-18e0dbb5a684","requesterId":"","did":"a3e487025d29f5b2cd599a8817ac16b8f3776a63","key":""},"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499971432,"ver":"3.0","mid":"LOG:17ffd4c05d66e0aa0ed0c1b337192eae","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971436,"ver":"3.0","mid":"LOG:3946ef96e11ada0bec3722f68007850d","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499971438,"ver":"3.0","mid":"LOG:746341bbfac5363693478dff90e22123","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"START","ets":1529499971560,"ver":"3.0","mid":"START:21e01edc45ab176abfd316bc52a8a544","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"user","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"session","uaspec":{"agent":"Chrome","ver":"67.0.3396.79","system":"Mac OS","platform":"WebKit","raw":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"},"duration":1529499971560}},{"eid":"LOG","ets":1529499976065,"ver":"3.0","mid":"LOG:5da0c8e5582a85a4f2aabd6785bbdd91","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499976133,"ver":"3.0","mid":"LOG:db6a4ea5bf6aba1d6d3aaa4cc66b0071","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976136,"ver":"3.0","mid":"LOG:cdb9df09dee37b1488926b2c85402cc0","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499976138,"ver":"3.0","mid":"LOG:46eaf5cbdf7d748e311898df93ecec48","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499977167,"ver":"3.0","mid":"LOG:519081a41ee9f1550a889c81ff18ca83","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499977178,"ver":"3.0","mid":"LOG:a6e33b9c669acc8231d9eb372e51a330","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499977212,"ver":"3.0","mid":"LOG:634d18ebc34be27d9d2940b9526ced33","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987034,"ver":"3.0","mid":"LOG:c39e332ad0b7dc56c19cbc047184905a","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499987116,"ver":"3.0","mid":"LOG:6ac822896cd8a1736d55806c13ada64c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/user/v1/read/159e93d1-da0c-4231-be94-e75b0c226d7c?fields=completeness,missingFields,lastLoginTime"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987118,"ver":"3.0","mid":"LOG:97b4f69fc3fe8fead706060e67837caa","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/data/v1/role/read"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499987121,"ver":"3.0","mid":"LOG:df3cd7e874f2073d6d6b8ec4b3bf0b76","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"learner-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/learner/course/v1/user/enrollment/list/159e93d1-da0c-4231-be94-e75b0c226d7c"},{"protocol":"https"},{"method":"GET"},{}]}},{"eid":"LOG","ets":1529499988208,"ver":"3.0","mid":"LOG:1daf5f045d5d9e0a2b46f3fedbeb5d0c","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/org/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499988217,"ver":"3.0","mid":"LOG:2cd16668d217a92264a516c3e0e86709","actor":{"id":"public","type":"public"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"tenant","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"","cdata":[],"rollup":{"l1":"0123673542904299520","l2":"0123673689120112640","l3":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"Called tenant info","params":[]}},{"eid":"LOG","ets":1529499988252,"ver":"3.0","mid":"LOG:c1e597bce7abfbf31137079fa31ceeeb","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}},{"eid":"LOG","ets":1529499994398,"ver":"3.0","mid":"LOG:e20e25e5b1b58e77697242277d3d39bd","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/content/v1/read/do_1125232413877207041109?fields=createdBy,status,mimeType&mode=edit"},{"protocol":"https"},{"method":"GET"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json b/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json
deleted file mode 100644
index 059720429c..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/event_without_syncts.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":[{"eid":"LOG","ets":1529499971358,"ver":"3.0","mid":"LOG:5f3c177f90bd5833deade577cc28cbb6","actor":{"id":"159e93d1-da0c-4231-be94-e75b0c226d7c","type":"user"},"context":{"channel":"b00bc992ef25f1a9a8d63291e20efc8d","pdata":{"id":"local.sunbird.portal","ver":"0.0.1"},"env":"content-service","sid":"PCNHgbKZvh6Yis8F7BxiaJ1EGw0N3L9B","did":"cab2a0b55c79d12c8f0575d6397e5678","cdata":[],"rollup":{"l1":"ORG_001","l2":"0123673542904299520","l3":"0123673689120112640","l4":"b00bc992ef25f1a9a8d63291e20efc8d"}},"object":{},"tags":["b00bc992ef25f1a9a8d63291e20efc8d"],"edata":{"type":"api_access","level":"INFO","message":"","params":[{"url":"/content/composite/v1/search"},{"protocol":"https"},{"method":"POST"},{}]}}],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0"}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json b/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json
deleted file mode 100644
index b040d7751d..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/not_json_event.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"events":["testing events"],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json b/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json
deleted file mode 100644
index 1060389692..0000000000
--- a/data-pipeline/telemetry-extractor/src/test/resources/without_event_key.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"ekstep.telemetry","ver":"3.0","ets":1529500243591,"data":[],"mid":"56c0c430-748b-11e8-ae77-cd19397ca6b0","syncts":1529500243955}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-location-updater/pom.xml b/data-pipeline/telemetry-location-updater/pom.xml
deleted file mode 100644
index fe2bd49ebd..0000000000
--- a/data-pipeline/telemetry-location-updater/pom.xml
+++ /dev/null
@@ -1,206 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>telemetry-location-updater</artifactId>
-    <version>0.2.3</version>
-    <packaging>jar</packaging>
-    <name>TelemetryLocationUpdater</name>
-    <description>
-        Updates State District/City  and device information for all telemetry events
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.cassandra</groupId>
-            <artifactId>cassandra-driver-core</artifactId>
-            <version>3.1.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.squareup.okhttp3</groupId>
-            <artifactId>okhttp</artifactId>
-            <version>3.2.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml b/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml
deleted file mode 100644
index 76a938a35b..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/telemetry-location-updater.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:telemetry-location-updater</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties b/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties
deleted file mode 100644
index 6d4aac36f4..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/config/telemetry-location-updater.properties
+++ /dev/null
@@ -1,84 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.TelemetryLocationUpdater
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_location_updater_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.TelemetryLocationUpdaterTask
-task.inputs=kafka.__env__.telemetry.sink
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.success.topic.name=__env__.telemetry.with_location
-output.failed.topic.name=__env__.telemetry.failed
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,error-message-count,cache-hit-count,cache-miss-count,cache-empty-values-count,processed-message-count,unprocessed-message-count,user-cache-hit-count,user-declared-hit-count,ip-location-hit-count
-
-# redis
-# redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=2
-location.db.redis.key.expiry.seconds=__location_db_redis_key_expiry_seconds__
-cache.unresolved.location.key.expiry.seconds=__cache_unresolved_location_key_expiry_seconds__
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.userStore.id=4
-redis.database.deviceLocationStore.id=2
\ No newline at end of file
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java
deleted file mode 100644
index 788322232d..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/DeviceProfile.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-
-public class DeviceProfile {
-  private String countryCode;
-  private String country;
-  private String stateCode;
-  private String state;
-  private String city;
-  private String districtCustom;
-  private String stateCodeCustom;
-  private String stateCustomName;
-  private String userDeclaredState;
-  private String userDeclaredDistrict;
-  private Map<String, String> devicespec;
-  private Long firstaccess;
-  private Gson gson = new Gson();
-  private Type type = new TypeToken<Map<String, Object>>() {}.getType();
-
-  public DeviceProfile() {
-    this.countryCode = "";
-    this.country = "";
-    this.stateCode = "";
-    this.state = "";
-    this.city = "";
-    this.districtCustom = "";
-    this.stateCodeCustom = "";
-    this.stateCustomName = "";
-    this.userDeclaredState = "";
-    this.userDeclaredDistrict = "";
-    this.devicespec = new HashMap<>();
-    this.firstaccess = 0L;
-  }
-
-  public Map<String, String> toMap() {
-    Map<String, String> values = new HashMap<>();
-    values.put("country_code", DeviceProfile.getValueOrDefault(this.countryCode, ""));
-    values.put("country", DeviceProfile.getValueOrDefault(this.country, ""));
-    values.put("state_code", DeviceProfile.getValueOrDefault(this.stateCode, ""));
-    values.put("state", DeviceProfile.getValueOrDefault(this.state, ""));
-    values.put("city", DeviceProfile.getValueOrDefault(this.city, ""));
-    values.put("district_custom", DeviceProfile.getValueOrDefault(this.districtCustom, ""));
-    values.put("state_custom", DeviceProfile.getValueOrDefault(this.stateCustomName, ""));
-    values.put("state_code_custom", DeviceProfile.getValueOrDefault(this.stateCodeCustom, ""));
-    values.put("user_declared_state", DeviceProfile.getValueOrDefault(this.userDeclaredState, ""));
-    values.put("user_declared_district", DeviceProfile.getValueOrDefault(this.userDeclaredDistrict, ""));
-    values.put("devicespec", gson.toJson(DeviceProfile.getValueOrDefault(this.devicespec, new HashMap<>())));
-    values.put("firstaccess", DeviceProfile.getValueOrDefault(String.valueOf(this.firstaccess), ""));
-    return values;
-  }
-
-  public DeviceProfile fromMap(Map<String, String> map) {
-    this.countryCode = map.getOrDefault("country_code", "");
-    this.country = map.getOrDefault("country", "");
-    this.stateCode = map.getOrDefault("state_code", "");
-    this.state = map.getOrDefault("state", "");
-    this.city = map.getOrDefault("city", "");
-    this.districtCustom = map.getOrDefault("district_custom", "");
-    this.stateCustomName = map.getOrDefault("state_custom", "");
-    this.stateCodeCustom = map.getOrDefault("state_code_custom", "");
-    this.userDeclaredState = map.getOrDefault("user_declared_state", "");
-    this.userDeclaredDistrict = map.getOrDefault("user_declared_district", "");
-    this.devicespec = gson.fromJson(map.getOrDefault("devicespec", "{}"), type);
-    this.firstaccess = Long.valueOf(map.getOrDefault("firstaccess", "0"));
-    return this;
-  }
-
-  public DeviceProfile(String countryCode, String country, String stateCode, String state, String city) {
-    this.countryCode = countryCode;
-    this.country = country;
-    this.stateCode = stateCode;
-    this.state = state;
-    this.city = city;
-  }
-
-  public DeviceProfile(String countryCode, String country, String stateCode, String state,
-                       String city, String districtCustom, String stateCustomName,
-                       String stateCodeCustom, Map<String, String> device_spec, Long first_access,
-                       String userDeclaredDistrict, String userDeclaredState) {
-    this.countryCode = countryCode;
-    this.country = country;
-    this.stateCode = stateCode;
-    this.state = state;
-    this.city = city;
-    this.districtCustom = districtCustom;
-    this.stateCustomName = stateCustomName;
-    this.stateCodeCustom = stateCodeCustom;
-    this.userDeclaredState = userDeclaredState;
-    this.userDeclaredDistrict = userDeclaredDistrict;
-    this.devicespec = device_spec;
-    this.firstaccess = first_access;
-  }
-
-  public String getCity() {
-    return this.city;
-  }
-
-  public String getState() {
-    return this.state;
-  }
-
-  public String getCountryCode() {
-    return countryCode;
-  }
-
-  public String getCountry() {
-    return country;
-  }
-
-  public String getStateCode() {
-    return stateCode;
-  }
-
-  public String getDistrictCustom() {
-    return districtCustom;
-  }
-
-  public String getstateCustomName() {
-    return stateCustomName;
-  }
-
-  public String getstateCodeCustom() {
-    return stateCodeCustom;
-  }
-
-  public String getUserDeclaredState() { return userDeclaredState; }
-
-  public String getUserDeclaredDistrict() { return userDeclaredDistrict; }
-
-  public Map getDevicespec() {
-    return devicespec;
-  }
-
-  public Long getFirstaccess() {
-    return firstaccess;
-  }
-
-  public Boolean isLocationResolved() {
-    return this.state != null && !this.state.isEmpty();
-  }
-
-  public Boolean isDeviceProfileResolved() {
-    return this.isLocationResolved() || (!this.devicespec.isEmpty() || this.firstaccess > 0);
-  }
-
-  public static <T> T getValueOrDefault(T value, T defaultValue) {
-    return value == null ? defaultValue : value;
-  }
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 0ea74f15ac..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,83 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonObject;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig;
-
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    private Gson gson = new Gson();
-    private Type type = new TypeToken<Map<String, Object>>() {
-    }.getType();
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public void markFailure(String error, TelemetryLocationUpdaterConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tr_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("metadata.tr_error", error);
-        telemetry.add("metadata.src", config.jobName());
-    }
-
-    public void addDeviceProfile(DeviceProfile deviceProfile) {
-        Map<String, Object> ldata = new HashMap<>();
-
-        ldata.put("countrycode", deviceProfile.getCountryCode());
-        ldata.put("country", deviceProfile.getCountry());
-        ldata.put("statecode", deviceProfile.getStateCode());
-        ldata.put("state", deviceProfile.getState());
-        ldata.put("city", deviceProfile.getCity());
-        ldata.put("statecustomcode", deviceProfile.getstateCodeCustom());
-        ldata.put("statecustomname", deviceProfile.getstateCustomName());
-        ldata.put("districtcustom", deviceProfile.getDistrictCustom());
-        ldata.put("devicespec", deviceProfile.getDevicespec());
-        ldata.put("firstaccess", deviceProfile.getFirstaccess());
-        String iso3166statecode = addISOStateCodeToDeviceProfile(deviceProfile);
-        if (!iso3166statecode.isEmpty()) {
-            ldata.put("iso3166statecode", iso3166statecode);
-        }
-
-        Map<String, String> userDeclared = new HashMap<>();
-        userDeclared.put("state", deviceProfile.getUserDeclaredState());
-        userDeclared.put("district", deviceProfile.getUserDeclaredDistrict());
-        ldata.put("userdeclared", userDeclared);
-        telemetry.add(path.deviceData(), ldata);
-    }
-
-    public void addDerivedLocation(Map<String, String> derivedData) {
-        telemetry.add(path.derivedLocationData(), derivedData);
-    }
-
-    public String addISOStateCodeToDeviceProfile(DeviceProfile deviceProfile) {
-        // add new statecode field
-        String statecode = deviceProfile.getStateCode();
-        if (statecode != null && !statecode.isEmpty()) {
-            return "IN-" + statecode;
-        } else return "";
-    }
-
-    public void removeEdataLoc() {
-        Gson gson = new Gson();
-        JsonObject json = gson.toJsonTree(getMap()).getAsJsonObject().get("edata").getAsJsonObject();
-        json.remove("loc");
-        telemetry.add(path.edata(), json);
-    }
-
-    public void setFlag(String key, Object value) {
-        NullableValue<Map<String, Object>> telemetryFlag = telemetry.read(path.flags());
-        Map<String, Object> flags = telemetryFlag.isNull() ? new HashMap<>() : telemetryFlag.value();
-        flags.put(key, value);
-        telemetry.add(path.flags(), flags);
-    }
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java
deleted file mode 100644
index b894ccc2ee..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/service/TelemetryLocationUpdaterService.java
+++ /dev/null
@@ -1,176 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.events.domain.Path;
-import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig;
-import org.ekstep.ep.samza.task.TelemetryLocationUpdaterSink;
-import org.ekstep.ep.samza.task.TelemetryLocationUpdaterSource;
-import org.ekstep.ep.samza.util.DeviceProfileCache;
-
-import org.ekstep.ep.samza.util.RedisConnect;
-import redis.clients.jedis.Jedis;
-import com.google.gson.Gson;
-import redis.clients.jedis.exceptions.JedisException;
-
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TelemetryLocationUpdaterService {
-
-	private static Logger LOGGER = new Logger(TelemetryLocationUpdaterService.class);
-	private DeviceProfileCache deviceProfileCache;
-	private int userStoreDb;
-	private RedisConnect redisConnect;
-	private Jedis userDataStoreConnection;
-	private JobMetrics metrics;
-	private Gson gson = new Gson();
-	private Path path = new Path();
-	private Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-
-	public TelemetryLocationUpdaterService(DeviceProfileCache deviceLocationCache, JobMetrics metrics, RedisConnect redisConnect, Config config) {
-		this.deviceProfileCache = deviceLocationCache;
-		this.metrics = metrics;
-		this.userStoreDb = config.getInt("redis.database.userStore.id", 4);
-		this.redisConnect = redisConnect;
-		this.userDataStoreConnection = this.redisConnect.getConnection(userStoreDb);
-	}
-
-	public void process(TelemetryLocationUpdaterSource source, TelemetryLocationUpdaterSink sink) {
-		try {
-			Event event = source.getEvent();
-			// Temporary fix for skipping pipeline metrics
-			if (null != event.eid()) {
-				String did = event.did();
-				DeviceProfile deviceProfile = null;
-				if (did != null && !did.isEmpty()) {
-					// get device profile from cache
-					deviceProfile = deviceProfileCache.getDeviceProfileForDeviceId(did);
-
-					// check for user profile location
-					Map<String, String> derivedLocation = getLocationFromUserCache(event);
-
-					// get user declared location if user profile location is empty
-					if (derivedLocation.isEmpty()) derivedLocation = getUserDeclaredLocation(deviceProfile);
-
-					// get ip resolved location if user declared location is empty
-					if (derivedLocation.isEmpty()) derivedLocation = getIpResolvedLocation(deviceProfile);
-
-					// Add derived location to telemetry
-					if (derivedLocation.isEmpty()) {
-						metrics.incCacheMissCounter();
-						event.setFlag(TelemetryLocationUpdaterConfig.getDerivedLocationJobFlag(), false);
-					} else {
-						// update derived location metrics
-						updateDerivedLocationMetrics(derivedLocation);
-						event.addDerivedLocation(derivedLocation);
-						event.setFlag(TelemetryLocationUpdaterConfig.getDerivedLocationJobFlag(), true);
-					}
-
-					// Add device profile details to the event
-					updateEvent(event, deviceProfile);
-					metrics.incProcessedMessageCount();
-					sink.toSuccessTopic(event);
-				} else {
-					updateEvent(event, deviceProfile);
-					metrics.incUnprocessedMessageCount();
-					sink.toSuccessTopic(event);
-				}
-			}
-		} catch (JsonSyntaxException ex) {
-			LOGGER.error(null, "INVALID EVENT: " + source.getMessage(), ex);
-			sink.toMalformedTopic(source.getMessage());
-		}
-	}
-
-	private Map<String, String> getLocationFromUserCache(Event event) {
-		String uid = event.actorId();
-		Map<String, String> locationData;
-		try {
-			locationData = getLocationForUser(uid);
-			return locationData;
-		}
-		catch (JedisException ex) {
-			LOGGER.error(null, "Reconnecting with Redis store due to exception: ", ex);
-			this.userDataStoreConnection.close();
-			this.userDataStoreConnection = redisConnect.getConnection(userStoreDb);
-			return getLocationForUser(uid);
-		}
-	}
-
-    private Map<String, String> getLocationForUser(String uid) {
-        Map<String, Object> userCacheData;
-        Map<String, String> locationData = new HashMap<>();
-        if (uid != null) {
-            String data = userDataStoreConnection.get(uid);
-            if (data != null && !data.isEmpty()) {
-                userCacheData = gson.fromJson(data, mapType);
-                if (!userCacheData.isEmpty() && userCacheData.containsKey("state") && !userCacheData.get("state").toString().isEmpty()) {
-                    locationData.put(path.stateKey(), userCacheData.get("state").toString());
-                    locationData.put(path.districtKey(), userCacheData.getOrDefault("district", "").toString());
-                    locationData.put(path.locDerivedFromKey(), "user-profile");
-                }
-            }
-        }
-        return locationData;
-    }
-
-	private Map<String, String> getUserDeclaredLocation(DeviceProfile deviceProfile) {
-		Map<String, String> locationData = new HashMap<>();
-		if (null != deviceProfile) {
-			Map<String, String> data = deviceProfile.toMap();
-			if (!data.isEmpty() && data.containsKey("user_declared_state") && !data.get("user_declared_state").isEmpty()) {
-				locationData.put(path.stateKey(), data.get("user_declared_state").toString());
-				locationData.put(path.districtKey(), data.getOrDefault("user_declared_district", "").toString());
-				locationData.put(path.locDerivedFromKey(), "user-declared");
-			}
-		}
-		return locationData;
-	}
-
-	private Map<String, String> getIpResolvedLocation(DeviceProfile deviceProfile) {
-		Map<String, String> locationData = new HashMap<>();
-		if (null != deviceProfile) {
-			Map<String,String> data = deviceProfile.toMap();
-			if(!data.isEmpty() && data.containsKey("state")) {
-				locationData.put(path.stateKey(), data.get("state").toString());
-				locationData.put(path.districtKey(), data.getOrDefault("district_custom", "").toString());
-				locationData.put(path.locDerivedFromKey(), "ip-resolved");
-			}
-		}
-		return locationData;
-	}
-
-	private void updateDerivedLocationMetrics(Map<String, String> derivedLocation) {
-		String derivedFrom = derivedLocation.get("from");
-		if("user-profile".equalsIgnoreCase(derivedFrom)) metrics.incUserCacheHitCount();
-		if("user-declared".equalsIgnoreCase(derivedFrom)) metrics.incUserDeclaredHitCount();
-		if("ip-resolved".equalsIgnoreCase(derivedFrom)) metrics.incIpLocationHitCount();
-	}
-
-	public void updateEvent(Event event, DeviceProfile deviceProfile) {
-		event.removeEdataLoc();
-		if (null != deviceProfile) {
-			event.addDeviceProfile(deviceProfile);
-			if (deviceProfile.isDeviceProfileResolved()) {
-				event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), true);
-			} else {
-				event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), false);
-			}
-
-			if (deviceProfile.isLocationResolved()) {
-				event.setFlag(TelemetryLocationUpdaterConfig.getDeviceLocationJobFlag(), true);
-			} else {
-				event.setFlag(TelemetryLocationUpdaterConfig.getDeviceLocationJobFlag(), false);
-			}
-		} else {
-			event.setFlag(TelemetryLocationUpdaterConfig.getDeviceProfileJobFlag(), false);
-		}
-	}
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java
deleted file mode 100644
index 904b63b5d0..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterConfig.java
+++ /dev/null
@@ -1,48 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-public class TelemetryLocationUpdaterConfig {
-
-    private final String JOB_NAME = "TelemetryLocationUpdater";
-    private static final String deviceProfileJobFlag = "device_profile_retrieved";
-    private static final String deviceLocationJobFlag = "device_location_retrieved";
-    private static final String derivedLocationJobFlag = "derived_location_retrieved";
-
-    private String successTopic;
-    private String failedTopic;
-    private String malformedTopic;
-
-    public TelemetryLocationUpdaterConfig(Config config) {
-    	
-        successTopic = config.get("output.success.topic.name", "telemetry.with_loation");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-
-    public static String getDeviceProfileJobFlag() { return deviceProfileJobFlag; }
-
-    public static String getDeviceLocationJobFlag() {
-        return deviceLocationJobFlag;
-    }
-
-    public static String getDerivedLocationJobFlag() {
-        return derivedLocationJobFlag;
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java
deleted file mode 100644
index 6d3ef99466..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSink.java
+++ /dev/null
@@ -1,34 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-
-public class TelemetryLocationUpdaterSink extends BaseSink {
-
-	private TelemetryLocationUpdaterConfig config;
-
-	public TelemetryLocationUpdaterSink(MessageCollector collector, JobMetrics metrics, TelemetryLocationUpdaterConfig config) {
-		super(collector, metrics);
-		this.config = config;
-	}
-
-	public void toSuccessTopic(Event event) {
-		toTopic(config.successTopic(), event.did(), event.getJson());
-		metrics.incSuccessCounter();
-	}
-
-	public void toErrorTopic(Event event, String errorMessage) {
-		event.markFailure(errorMessage, config);
-		toTopic(config.failedTopic(), event.did(), event.getJson());
-		metrics.incErrorCounter();
-	}
-
-	public void toMalformedTopic(String message) {
-		toTopic(config.malformedTopic(), null, message);
-		metrics.incErrorCounter();
-	}
-
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java
deleted file mode 100644
index 60edb1411b..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterSource.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import java.util.Map;
-
-public class TelemetryLocationUpdaterSource {
-    static Logger LOGGER = new Logger(TelemetryLocationUpdaterSource.class);
-    
-    private IncomingMessageEnvelope envelope;
-
-    public TelemetryLocationUpdaterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent(){
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-	private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java
deleted file mode 100644
index 5e5042cdcd..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTask.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.*;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.TelemetryLocationUpdaterService;
-import org.ekstep.ep.samza.util.DeviceProfileCache;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class TelemetryLocationUpdaterTask extends BaseSamzaTask {
-
-	private TelemetryLocationUpdaterConfig config;
-	private JobMetrics metrics;
-	private TelemetryLocationUpdaterService service;
-	private DeviceProfileCache deviceProfileCache;
-	private RedisConnect redisConnect;
-
-	public TelemetryLocationUpdaterTask(Config config, TaskContext context, DeviceProfileCache deviceProfileCache, RedisConnect redisConnect) {
-		init(config, context, deviceProfileCache, redisConnect);
-	}
-
-	public TelemetryLocationUpdaterTask() {
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		init(config, context, deviceProfileCache, redisConnect);
-	}
-
-	public void init(Config config, TaskContext context, DeviceProfileCache deviceProfileCache, RedisConnect redisConnect) {
-		
-		this.config = new TelemetryLocationUpdaterConfig(config);
-		this.metrics = new JobMetrics(context, this.config.jobName());
-		this.redisConnect = redisConnect == null ? new RedisConnect(config) : redisConnect;
-		this.deviceProfileCache = deviceProfileCache == null ? new DeviceProfileCache(config, metrics, this.redisConnect) : deviceProfileCache;
-		this.service = new TelemetryLocationUpdaterService(this.deviceProfileCache, metrics, this.redisConnect, config);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-						TaskCoordinator taskCoordinator) throws Exception {
-
-		TelemetryLocationUpdaterSink sink = new TelemetryLocationUpdaterSink(collector, metrics, config);
-		TelemetryLocationUpdaterSource source = new TelemetryLocationUpdaterSource(envelope);
-		service.process(source, sink);
-	}
-
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java b/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java
deleted file mode 100644
index 04b415bbf2..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/java/org/ekstep/ep/samza/util/DeviceProfileCache.java
+++ /dev/null
@@ -1,56 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-import java.util.Map;
-
-
-public class DeviceProfileCache {
-
-    private static Logger LOGGER = new Logger(DeviceProfileCache.class);
-    private RedisConnect redisConnect;
-    private JobMetrics metrics;
-    private Jedis redisConnection;
-    private int databaseIndex;
-
-    public DeviceProfileCache(Config config, JobMetrics metrics, RedisConnect redisConnect) {
-        this.databaseIndex = config.getInt("redis.database.deviceLocationStore.id", 2);
-        this.redisConnect = redisConnect;
-        this.redisConnection = this.redisConnect.getConnection(databaseIndex);
-        this.metrics = metrics;
-    }
-
-    public DeviceProfile getDeviceProfileForDeviceId(String did) {
-        DeviceProfile deviceProfile = null;
-        if (did != null && !did.isEmpty()) {
-            // Get location from redis. Safe retry once for redis connection loss or error
-            try {
-                deviceProfile = getDeviceProfileFromCache(did);
-            } catch (JedisException ex) {
-                LOGGER.error(null, "Reconnecting with Redis store due to exception: ", ex);
-                this.redisConnection.close();
-                this.redisConnection = redisConnect.getConnection(databaseIndex);
-                deviceProfile = getDeviceProfileFromCache(did);
-            }
-            if (null != deviceProfile && deviceProfile.isLocationResolved()) {
-                metrics.incCacheHitCounter();
-            } else {
-                metrics.incEmptyCacheValueCounter();
-            }
-        }
-        return deviceProfile;
-    }
-
-    public DeviceProfile getDeviceProfileFromCache(String deviceId) {
-        Map<String, String> data = redisConnection.hgetAll(deviceId);
-        if (data.size() > 0) {
-            return new DeviceProfile().fromMap(data);
-        } else {
-            return null;
-        }
-    }
-}
diff --git a/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml b/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/telemetry-location-updater/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java
deleted file mode 100644
index c338f3397e..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/UserLocationCacheTest.java
+++ /dev/null
@@ -1,22 +0,0 @@
-//package org.ekstep.ep.samza;
-//
-//import org.apache.samza.config.Config;
-//import org.ekstep.ep.samza.util.*;
-//import org.junit.Before;
-//import static org.mockito.Mockito.*;
-//
-//public class UserLocationCacheTest {
-//
-//    private UserLocationCache userLocationCache;
-//    private RedisConnect redisConnectMock;
-//    private CassandraConnect cassandraConnectMock;
-//
-//    @SuppressWarnings("unchecked")
-//    @Before
-//    public void setUp() {
-//        Config config = mock(Config.class);
-//        this.redisConnectMock = mock(RedisConnect.class);
-//        this.cassandraConnectMock = mock(CassandraConnect.class);
-//        this.userLocationCache = new UserLocationCache(config, redisConnectMock, cassandraConnectMock);
-//    }
-//}
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index 78c4e91709..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,469 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.util.Map;
-
-public class EventFixture {
-
-	public static final String LOG_EVENT = "{\n" +
-            "  \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"data\": \"\",\n" +
-            "      \"err\": \"10\",\n" +
-            "      \"eventId\": \"GE_SIGNUP\",\n" +
-            "      \"id\": \"2131165210\",\n" +
-            "      \"type\": \"GENIE\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"2.2.15\"\n" +
-            "  },\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"2.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-	public static final String ERROR_EVENT = "{\n" +
-            "  \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"eid\": \"ERROR\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"2.2.15\"\n" +
-            "  },\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"2.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-	public static final String UNPARSABLE_START_EVENT = "{\n" +
-            "  \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"mode\": \"WIFI\",\n" +
-            "      \"ver\": \"12\",\n" +
-            "      \"size\": 12.67,\n" +
-            "      \"err\": \"\",\n" +
-            "      \"referrer\": [\n" +
-            "        {\n" +
-            "          \"action\": \"INSTALL\",\n" +
-            "          \"utmsource\": \"Ekstep\",\n" +
-            "          \"utmmedium\": \"Portal\",\n" +
-            "          \"utmterm\": \"December 2016\",\n" +
-            "          \"utmcontent\": \"Ramayana\",\n" +
-            "          \"utmcampaign\": \"Epics of India\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"START\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"1.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.0\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    ";
-	public static final String START_EVENT = "{\n" +
-            "  \"cdata\": [],\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"extype\": \"\",\n" +
-            "      \"id\": \"\",\n" +
-            "      \"pos\": [],\n" +
-            "      \"stageid\": \"Genie-TelemetrySync\",\n" +
-            "      \"subtype\": \"ManualSync-Success\",\n" +
-            "      \"tid\": \"\",\n" +
-            "      \"type\": \"OTHER\",\n" +
-            "      \"uri\": \"\",\n" +
-            "      \"values\": [\n" +
-            "        {\n" +
-            "          \"SizeOfFileInKB\": \"0.81\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"GE_INTERACT\",\n" +
-            "  \"etags\": {\n" +
-            "    \"app\": []\n" +
-            "  },\n" +
-            "  \"ets\": 1506328824238,\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"org.ekstep.genieservices.qa\",\n" +
-            "    \"ver\": \"6.5.localdev-debug\"\n" +
-            "  },\n" +
-            "  \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"6.5.localdev-debug\"\n" +
-            "  },\n" +
-            "  \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" +
-            "  \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" +
-            "  \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"@version\": \"1\",\n" +
-            "  \"@timestamp\": \"2017-09-25T08:35:44.037Z\",\n" +
-            "  \"metadata\": {\n" +
-            "    \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" +
-            "  },\n" +
-            "  \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" +
-            "  \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-            "  \"type\": \"events\",\n" +
-            "  \"flags\": {\n" +
-            "    \"dd_processed\": true\n" +
-            "  }\n" +
-            "}";
-
-	public static final String INTERACT_EVENT = "{\n" +
-			"    \"actor\": {\n" +
-			"        \"type\": \"User\",\n" +
-			"        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-			"    },\n" +
-			"    \"eid\": \"INTERACT\",\n" +
-			"    \"edata\": {\n" +
-			"		 \"loc\":\"xyz\",\n" +
-			"        \"id\": \"ContentDetail\",\n" +
-			"        \"pageid\": \"ContentDetail\",\n" +
-			"        \"type\": \"TOUCH\",\n" +
-			"        \"subtype\": \"ContentDownload-Initiate\"\n" +
-			"    },\n" +
-			"    \"ver\": \"3.0\",\n" +
-			"    \"ets\": 1541574545180,\n" +
-			"    \"context\": {\n" +
-			"        \"pdata\": {\n" +
-			"            \"ver\": \"2.1.8\",\n" +
-			"            \"pid\": \"sunbird.app\",\n" +
-			"            \"id\": \"prod.diksha.app\"\n" +
-			"        },\n" +
-			"        \"channel\": \"0123221617357783046602\",\n" +
-			"        \"env\": \"sdk\",\n" +
-			"        \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-			"        \"cdata\": [\n" +
-			"            {\n" +
-			"                \"type\": \"qr\",\n" +
-			"                \"id\": \"K4KCXE\"\n" +
-			"            },\n" +
-			"            {\n" +
-			"                \"type\": \"API\",\n" +
-			"                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-			"            }\n" +
-			"        ],\n" +
-			"        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-			"    },\n" +
-			"    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-			"    \"object\": {\n" +
-			"        \"id\": \"do_31249561779090227216256\",\n" +
-			"        \"type\": \"Content\",\n" +
-			"        \"version\": \"\"\n" +
-			"    },\n" +
-			"    \"tags\": [\n" +
-			"        \n" +
-			"    ],\n" +
-			"    \"syncts\": 1539846605341,\n" +
-			"    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-			"}";
-
-	public static final String INTERACT_EVENT_WITHOUT_DID = "{\n" +
-			"    \"actor\": {\n" +
-			"        \"type\": \"User\",\n" +
-			"        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-			"    },\n" +
-			"    \"eid\": \"INTERACT\",\n" +
-			"    \"edata\": {\n" +
-			"		 \"loc\":\"xyz\",\n" +
-			"        \"id\": \"ContentDetail\",\n" +
-			"        \"pageid\": \"ContentDetail\",\n" +
-			"        \"type\": \"TOUCH\",\n" +
-			"        \"subtype\": \"ContentDownload-Initiate\"\n" +
-			"    },\n" +
-			"    \"ver\": \"3.0\",\n" +
-			"    \"ets\": 1541574545180,\n" +
-			"    \"context\": {\n" +
-			"        \"pdata\": {\n" +
-			"            \"ver\": \"2.1.8\",\n" +
-			"            \"pid\": \"sunbird.app\",\n" +
-			"            \"id\": \"prod.diksha.app\"\n" +
-			"        },\n" +
-			"        \"channel\": \"0123221617357783046602\",\n" +
-			"        \"env\": \"sdk\",\n" +
-			"        \"cdata\": [\n" +
-			"            {\n" +
-			"                \"type\": \"qr\",\n" +
-			"                \"id\": \"K4KCXE\"\n" +
-			"            },\n" +
-			"            {\n" +
-			"                \"type\": \"API\",\n" +
-			"                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-			"            }\n" +
-			"        ],\n" +
-			"        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-			"    },\n" +
-			"    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-			"    \"object\": {\n" +
-			"        \"id\": \"do_31249561779090227216256\",\n" +
-			"        \"type\": \"Content\",\n" +
-			"        \"version\": \"\"\n" +
-			"    },\n" +
-			"    \"tags\": [\n" +
-			"        \n" +
-			"    ],\n" +
-			"    \"syncts\": 1539846605341,\n" +
-			"    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-			"}";
-
-	public static final String INTERACT_EVENT_WITH_ACTOR_AS_SYSTEM = "{\n" +
-			"    \"actor\": {\n" +
-			"        \"type\": \"System\",\n" +
-			"        \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-			"    },\n" +
-			"    \"eid\": \"INTERACT\",\n" +
-			"    \"edata\": {\n" +
-			"		 \"loc\":\"xyz\",\n" +
-			"        \"id\": \"ContentDetail\",\n" +
-			"        \"pageid\": \"ContentDetail\",\n" +
-			"        \"type\": \"TOUCH\",\n" +
-			"        \"subtype\": \"ContentDownload-Initiate\"\n" +
-			"    },\n" +
-			"    \"ver\": \"3.0\",\n" +
-			"    \"ets\": 1541574545180,\n" +
-			"    \"context\": {\n" +
-			"        \"pdata\": {\n" +
-			"            \"ver\": \"2.1.8\",\n" +
-			"            \"pid\": \"sunbird.app\",\n" +
-			"            \"id\": \"prod.diksha.app\"\n" +
-			"        },\n" +
-			"        \"channel\": \"0123221617357783046602\",\n" +
-			"        \"env\": \"sdk\",\n" +
-			"        \"cdata\": [\n" +
-			"            {\n" +
-			"                \"type\": \"qr\",\n" +
-			"                \"id\": \"K4KCXE\"\n" +
-			"            },\n" +
-			"            {\n" +
-			"                \"type\": \"API\",\n" +
-			"                \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-			"            }\n" +
-			"        ],\n" +
-			"        \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-			"    },\n" +
-			"    \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-			"    \"object\": {\n" +
-			"        \"id\": \"do_31249561779090227216256\",\n" +
-			"        \"type\": \"Content\",\n" +
-			"        \"version\": \"\"\n" +
-			"    },\n" +
-			"    \"tags\": [\n" +
-			"        \n" +
-			"    ],\n" +
-			"    \"syncts\": 1539846605341,\n" +
-			"    \"@timestamp\": \"2018-10-18T07:10:05.341Z\"\n" +
-			"}";
-
-	public static final String ANY_STRING = "Hey Samza, Whats Up?";
-	public static final String EMPTY_JSON = "{}";
-
-	public static final String CHANNEL_RESPONSE_BODY = "{\n" +
-			"    \"id\": \"api.org.search\",\n" +
-			"    \"ver\": \"v1\",\n" +
-			"    \"ts\": \"2019-01-08 06:40:12:316+0000\",\n" +
-			"    \"params\": {\n" +
-			"        \"resmsgid\": null,\n" +
-			"        \"msgid\": \"0f93ff40-582f-4599-a718-7bd898cd8b5d\",\n" +
-			"        \"err\": null,\n" +
-			"        \"status\": \"success\",\n" +
-			"        \"errmsg\": null\n" +
-			"    },\n" +
-			"    \"responseCode\": \"OK\",\n" +
-			"    \"result\": {\n" +
-			"        \"response\": {\n" +
-			"            \"count\": 1,\n" +
-			"            \"content\": [\n" +
-			"                {\n" +
-			"                    \"dateTime\": null,\n" +
-			"                    \"preferredLanguage\": \"English\",\n" +
-			"                    \"approvedBy\": null,\n" +
-			"                    \"channel\": \"ROOT_ORG\",\n" +
-			"                    \"description\": \"Andhra State Boardsssssss\",\n" +
-			"                    \"updatedDate\": \"2018-11-28 10:00:08:675+0000\",\n" +
-			"                    \"addressId\": null,\n" +
-			"                    \"provider\": null,\n" +
-			"                    \"locationId\": null,\n" +
-			"                    \"orgCode\": \"sunbird\",\n" +
-			"                    \"theme\": null,\n" +
-			"                    \"id\": \"ORG_001\",\n" +
-			"                    \"communityId\": null,\n" +
-			"                    \"isApproved\": null,\n" +
-			"                    \"email\": \"support_dev@sunbird.org\",\n" +
-			"                    \"slug\": \"sunbird\",\n" +
-			"                    \"identifier\": \"ORG_001\",\n" +
-			"                    \"thumbnail\": null,\n" +
-			"                    \"orgName\": \"Sunbird\",\n" +
-			"                    \"updatedBy\": \"1d7b85b0-3502-4536-a846-d3a51fd0aeea\",\n" +
-			"                    \"locationIds\": [\n" +
-			"                        \"969dd3c1-4e98-4c17-a994-559f2dc70e18\"\n" +
-			"                    ],\n" +
-			"                    \"externalId\": null,\n" +
-			"                    \"isRootOrg\": true,\n" +
-			"                    \"rootOrgId\": \"ORG_001\",\n" +
-			"                    \"approvedDate\": null,\n" +
-			"                    \"imgUrl\": null,\n" +
-			"                    \"homeUrl\": null,\n" +
-			"                    \"orgTypeId\": null,\n" +
-			"                    \"isDefault\": true,\n" +
-			"                    \"contactDetail\": \"[{\\\"phone\\\":\\\"213124234234\\\",\\\"email\\\":\\\"test@test.com\\\"},{\\\"phone\\\":\\\"+91213124234234\\\",\\\"email\\\":\\\"test1@test.com\\\"}]\",\n" +
-			"                    \"createdDate\": null,\n" +
-			"                    \"createdBy\": null,\n" +
-			"                    \"parentOrgId\": null,\n" +
-			"                    \"hashTagId\": \"b00bc992ef25f1a9a8d63291e20efc8d\",\n" +
-			"                    \"noOfMembers\": 5,\n" +
-			"                    \"status\": 1\n" +
-			"                }\n" +
-			"            ]\n" +
-			"        }\n" +
-			"    }\n" +
-			"}";
-
-	public static final String LOCATION_SEARCH_RESPONSE_BODY = "{\n" +
-			"    \"id\": \"api.location.search\",\n" +
-			"    \"ver\": \"v1\",\n" +
-			"    \"ts\": \"2019-01-08 06:10:57:676+0000\",\n" +
-			"    \"params\": {\n" +
-			"        \"resmsgid\": null,\n" +
-			"        \"msgid\": \"bddca208-34b5-4054-9a05-24eecfc12c99\",\n" +
-			"        \"err\": null,\n" +
-			"        \"status\": \"success\",\n" +
-			"        \"errmsg\": null\n" +
-			"    },\n" +
-			"    \"responseCode\": \"OK\",\n" +
-			"    \"result\": {\n" +
-			"        \"response\": [\n" +
-			"            {\n" +
-			"                \"code\": \"29\",\n" +
-			"                \"name\": \"Karnataka\",\n" +
-			"                \"id\": \"969dd3c1-4e98-4c17-a994-559f2dc70e18\",\n" +
-			"                \"type\": \"state\"\n" +
-			"            }\n" +
-			"        ]\n" +
-			"    }\n" +
-			"}";
-
-	public static final String LOCATION_SEARCH_UNSUCCESSFUL_RESPONSE = "{\n"+
-			"    \"id\": \"api.location.search\",\n"+
-			"    \"ver\": \"v1\",\n"+
-			"    \"ts\": \"2019-01-08 07:22:25:891+0000\",\n"+
-			"    \"params\": {\n"+
-			"        \"resmsgid\": null,\n"+
-			"        \"msgid\": \"15b23224-d7b5-4645-bbbe-3fd1172b4112\",\n"+
-			"        \"err\": null,\n"+
-			"        \"status\": \"success\",\n"+
-			"        \"errmsg\": null\n"+
-			"    },\n"+
-			"    \"responseCode\": \"OK\",\n"+
-			"    \"result\": {\n"+
-			"        \"response\": []\n"+
-			"    }\n"+
-			"}";
-
-	public static final String DEVICE_SUMMARY = "{\"eid\":\"ME_DEVICE_SUMMARY\",\"ets\":1573607175904,\"syncts\":1573557231120,\"ver\":\"1.0\",\"mid\":\"2D03E6A051F77F4EEE1F48E4AD7FE575\",\"context\":{\"pdata\":{\"id\":\"AnalyticsDataPipeline\",\"ver\":\"1.0\",\"model\":\"DeviceSummary\"},\"granularity\":\"DAY\",\"date_range\":{\"from\":1573196386089,\"to\":1573196390938},\"channel\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"dimensions\":{\"did\":\"099988ce86c4dbb9a4057ff611d38427\",\"channel\":\"b00bc992ef25f1a9a8d63291e20efc8d\"},\"edata\":{\"eks\":{\"firstAccess\":1573196380482,\"dial_stats\":{\"total_count\":0,\"success_count\":0,\"failure_count\":0},\"content_downloads\":0,\"contents_played\":0,\"total_ts\":4.85,\"total_launches\":1,\"unique_contents_played\":0}},\"flags\":{\"derived_dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-13T01:06:15.904+0000\"}";
-
-	public static final String CHANNEL_SEARCH_EMPTY_LOCATIONIDS_RESPONSE = "{\n" +
-			"    \"id\": \"api.org.search\",\n" +
-			"    \"ver\": \"v1\",\n" +
-			"    \"ts\": \"2019-01-08 08:57:47:192+0000\",\n" +
-			"    \"params\": {\n" +
-			"        \"resmsgid\": null,\n" +
-			"        \"msgid\": \"7b1cf31f-0b9f-4d88-b405-4d9c61be8081\",\n" +
-			"        \"err\": null,\n" +
-			"        \"status\": \"success\",\n" +
-			"        \"errmsg\": null\n" +
-			"    },\n" +
-			"    \"responseCode\": \"OK\",\n" +
-			"    \"result\": {\n" +
-			"        \"response\": {\n" +
-			"            \"count\": 1,\n" +
-			"            \"content\": [\n" +
-			"                {\n" +
-			"                    \"dateTime\": null,\n" +
-			"                    \"preferredLanguage\": \"English\",\n" +
-			"                    \"approvedBy\": null,\n" +
-			"                    \"channel\": \"sunbird-staging\",\n" +
-			"                    \"description\": \"default user will be associated with this\",\n" +
-			"                    \"updatedDate\": null,\n" +
-			"                    \"addressId\": null,\n" +
-			"                    \"orgType\": null,\n" +
-			"                    \"provider\": null,\n" +
-			"                    \"locationId\": null,\n" +
-			"                    \"orgCode\": \"defaultRootOrg\",\n" +
-			"                    \"theme\": null,\n" +
-			"                    \"id\": \"0125134851644620800\",\n" +
-			"                    \"communityId\": null,\n" +
-			"                    \"isApproved\": null,\n" +
-			"                    \"slug\": \"sunbird-staging\",\n" +
-			"                    \"identifier\": \"0125134851644620800\",\n" +
-			"                    \"thumbnail\": null,\n" +
-			"                    \"orgName\": \"defaultRootOrg\",\n" +
-			"                    \"updatedBy\": null,\n" +
-			"                    \"locationIds\": [],\n" +
-			"                    \"externalId\": null,\n" +
-			"                    \"isRootOrg\": true,\n" +
-			"                    \"rootOrgId\": \"0125134851644620800\",\n" +
-			"                    \"approvedDate\": null,\n" +
-			"                    \"imgUrl\": null,\n" +
-			"                    \"homeUrl\": null,\n" +
-			"                    \"orgTypeId\": null,\n" +
-			"                    \"isDefault\": null,\n" +
-			"                    \"contactDetail\": [],\n" +
-			"                    \"createdDate\": \"2018-05-28 16:23:38:330+0000\",\n" +
-			"                    \"createdBy\": \"8217108a-6931-491c-9009-1ae95cb0477f\",\n" +
-			"                    \"parentOrgId\": null,\n" +
-			"                    \"hashTagId\": \"0125134851644620800\",\n" +
-			"                    \"noOfMembers\": null,\n" +
-			"                    \"status\": 1\n" +
-			"                }\n" +
-			"            ]\n" +
-			"        }\n" +
-			"    }\n" +
-			"}";
-
-	public static Map<String, Object> getMap(String message) {
-		return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-	}
-	
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java
deleted file mode 100644
index 616206d3bb..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/DeviceProfileTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class DeviceProfileTest {
-
-    /**
-     * Method should return the city, country, countryCode
-     */
-    @Test
-    public void shouldReturnValues() {
-
-        DeviceProfile df = new DeviceProfile("IN", "India", "KA", "Karnatka", "Bengaluru");
-        DeviceProfile df2 = new DeviceProfile("IN", "India", "KA", "Karnatka", "Bengaluru");
-        Assert.assertEquals("Bengaluru", df.getCity());
-        Assert.assertEquals("India", df.getCountry());
-        Assert.assertEquals("IN", df.getCountryCode());
-        Assert.assertEquals("KA", df.getStateCode());
-    }
-
-
-    @Test
-    public void deviceProfileValuesShouldBeEmpty() {
-        DeviceProfile df = new DeviceProfile();
-        Assert.assertEquals("", df.getCity());
-        Assert.assertEquals("", df.getCountry());
-        Assert.assertEquals("", df.getCountryCode());
-        Assert.assertEquals("", df.getStateCode());
-        Assert.assertEquals("", df.getDistrictCustom());
-        Assert.assertEquals("", df.getstateCodeCustom());
-        Assert.assertEquals("", df.getUserDeclaredState());
-        Assert.assertEquals("", df.getstateCustomName());
-
-    }
-
-    @Test
-    public void deviceProfileValuesShouldAddTheValuesToCache() {
-        ObjectMapper mapper = new ObjectMapper();
-        String json = "{\"country_code\":\"IN\",\"country\":\"India\",\"state_code\":\"KA\",\"state\":\"Karnataka\",\"city\":\"Bengaluru\",\"user_declared_state\":\"Karnataka\",\"user_declared_district\":\"Tumkur\",\"district_custom\":\"BA\"}";
-        try {
-
-            // convert JSON string to Map
-            Map<String, String> values = mapper.readValue(json, Map.class);
-            DeviceProfile df = new DeviceProfile();
-            df.fromMap(values);
-
-            Assert.assertEquals("Bengaluru", df.getCity());
-            Assert.assertEquals("India", df.getCountry());
-            Assert.assertEquals("IN", df.getCountryCode());
-            Assert.assertEquals("KA", df.getStateCode());
-            Assert.assertEquals("BA", df.getDistrictCustom());
-            Assert.assertEquals("Karnataka", df.getUserDeclaredState());
-            Assert.assertEquals("Tumkur", df.getUserDeclaredDistrict());
-
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
-
-
-    }
-
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java
deleted file mode 100644
index c550fcfff5..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/system/EventTest.java
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-import java.util.Map;
-
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.task.TelemetryLocationUpdaterConfig;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-import org.mockito.Mockito;
-
-public class EventTest {
-	private TelemetryLocationUpdaterConfig configMock;
-	@Test
-	public void shouldReturnMid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT));
-		Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid());
-	}
-
-	@Test
-	public void shouldReturnEid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.LOG_EVENT));
-		Assert.assertEquals("LOG", event.eid());
-	}
-
-	@Test(expected = JsonSyntaxException.class)
-	public void shouldThrowException() {
-		new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT,
-				new TypeToken<Map<String, Object>>() {
-				}.getType());
-	}
-
-	@Test
-	public void shouldMarkFailure() {
-		configMock = Mockito.mock(TelemetryLocationUpdaterConfig.class);
-		String RAW_EVENT = "{\"eid\":\"INTERACT\",\"ets\":1573811794043,\"ver\":\"3.0\",\"mid\":\"INTERACT:dfdb7f3e3e5854a9a4b01d20e2ade835\",\"actor\":{\"id\":\"0b96635f-fe2b-4ab0-a511-05cfce8faa3f\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.5.0\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"0ITT0p3ZqwkxREhxTmCiQatUSWGisRpw\",\"did\":\"a3cf6d00e1b7af06a61300b4a50853fb\",\"cdata\":[{\"type\":\"Feature\",\"id\":\"video:resolutionChange\"},{\"type\":\"Task\",\"id\":\"SB-13358\"},{\"type\":\"Resolution\",\"id\":\"large\"},{\"type\":\"ResolutionChange\",\"id\":\"Auto\"},{\"id\":\"9d9c3e9aa3eb33090b61ca8db196f8e6\",\"type\":\"ContentSession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_312579855868370944110877\",\"type\":\"Content\",\"ver\":\"1\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"type\":\"TOUCH\",\"subtype\":\"CHANGE\",\"id\":\"\",\"pageid\":\"videostage\"}}";
-		Event event = new Event(new Gson().fromJson(RAW_EVENT, Map.class));
-		event.markFailure("Invalid Event", configMock);
-		Map<String, String> flagData = new Gson().fromJson(new Gson().toJson(event.getMap().get("flags")), Map.class);
-		Assert.assertNotNull(flagData);
-		Assert.assertEquals(flagData.get("tr_processed"), false);
-		Object metaData = event.getMap().get("metadata");
-		Assert.assertNotNull(metaData);
-	}
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java
deleted file mode 100644
index 22a08fd0e4..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/task/TelemetryLocationUpdaterTaskTest.java
+++ /dev/null
@@ -1,385 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import org.apache.commons.lang.StringUtils;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.DeviceProfileCache;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import redis.clients.jedis.Jedis;
-import com.fiftyonred.mock_jedis.MockJedis;
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class TelemetryLocationUpdaterTaskTest {
-
-	private static final String SUCCESS_TOPIC = "telemetry.with_location";
-	private static final String FAILED_TOPIC = "telemetry.failed";
-	private static final String MALFORMED_TOPIC = "telemetry.malformed";
-
-	private MessageCollector collectorMock;
-	private TaskCoordinator coordinatorMock;
-	private IncomingMessageEnvelope envelopeMock;
-	private DeviceProfileCache deviceProfileCacheMock;
-	private TelemetryLocationUpdaterTask telemetryLocationUpdaterTask;
-	private Jedis jedisMock = new MockJedis("test");
-
-	@SuppressWarnings("unchecked")
-	@Before
-	public void setUp() {
-		collectorMock = mock(MessageCollector.class);
-		TaskContext contextMock = mock(TaskContext.class);
-		MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
-		Counter counter = mock(Counter.class);
-		coordinatorMock = mock(TaskCoordinator.class);
-		envelopeMock = mock(IncomingMessageEnvelope.class);
-		Config configMock = mock(Config.class);
-		RedisConnect redisConnectMock = mock(RedisConnect.class);
-
-		deviceProfileCacheMock = mock(DeviceProfileCache.class);
-
-		stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC);
-		stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-		stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-		stub(configMock.getInt("redis.database.userStore.id", 4)).toReturn(4);
-
-		stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-		stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-		stub(envelopeMock.getOffset()).toReturn("2");
-		stub(envelopeMock.getSystemStreamPartition())
-				.toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1)));
-		stub(redisConnectMock.getConnection(4)).toReturn(jedisMock);
-
-		telemetryLocationUpdaterTask = new TelemetryLocationUpdaterTask(configMock, contextMock, deviceProfileCacheMock, redisConnectMock);
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicIfDidIsNull() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT_WITHOUT_DID);
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(null);
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() {
-		}.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				assertEquals("3.0", outputEvent.get("ver"));
-				assertFalse(outputEvent.containsKey("devicedata"));
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(false, flags.get("device_profile_retrieved"));
-				return true;
-			}
-		}));
-
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicWithUserProfileLocationAsDerived() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-		// Map<String, String> uaspec = new HashMap<>();
-		Map<String, String> device_spec = new HashMap<>();
-		Long first_access = 1559484698000L;
-		device_spec.put("os", "Android 6.0");
-		device_spec.put("make", "Motorola XT1706");
-		/*
-		uaspec.put("agent", "Mozilla");
-		uaspec.put("ver", "5.0");
-		uaspec.put("system", "iPad");
-		uaspec.put("platform", "AppleWebKit/531.21.10");
-		uaspec.put("raw", "Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)");
-		*/
-		DeviceProfile deviceProfile = new DeviceProfile
-				("IN", "India", "KA", "Karnataka",
-						"Bangalore", "Banglore-Custom", "Karnatak-Custom",
-						"KA-Custom", device_spec, first_access,"Bangalore",
-						"Karnataka");
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile);
-
-		jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}");
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				assertEquals("3.0", outputEvent.get("ver"));
-				Map<String, Object> devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType);
-
-				assertEquals("Karnataka", devicedata.get("state"));
-				assertEquals("Banglore-Custom", devicedata.get("districtcustom"));
-				assertEquals("IN", devicedata.get("countrycode"));
-				assertEquals("India", devicedata.get("country"));
-				assertEquals("KA", devicedata.get("statecode"));
-				assertEquals("Bangalore", devicedata.get("city"));
-				assertEquals("KA-Custom", devicedata.get("statecustomcode"));
-				assertEquals("Karnatak-Custom", devicedata.get("statecustomname"));
-				assertEquals("IN-KA", devicedata.get("iso3166statecode"));
-//				assertEquals(Long.valueOf("1559484698000"), Long.valueOf(devicedata.get("firstaccess").toString()));
-
-				Map<String, Object> deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType);
-				assertEquals("Android 6.0", deviceSpec.get("os"));
-				assertEquals("Motorola XT1706", deviceSpec.get("make"));
-
-				/*
-				Map<String, Object> uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType);
-				assertEquals("Mozilla", uaSpec.get("agent"));
-				assertEquals("5.0", uaSpec.get("ver"));
-				assertEquals("iPad", uaSpec.get("system"));
-				assertEquals("AppleWebKit/531.21.10", uaSpec.get("platform"));
-				assertEquals("Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)", uaSpec.get("raw"));
-				*/
-
-				Map<String, Object> userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType);
-				assertEquals("Karnataka", userDeclared.get("state"));
-				assertEquals("Bangalore", userDeclared.get("district"));
-
-				Map<String, Object> derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType);
-				assertEquals("Karnataka", derivedLocationData.get("state"));
-				assertEquals("Bengaluru", derivedLocationData.get("district"));
-				assertEquals("user-profile", derivedLocationData.get("from"));
-
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(true, flags.get("device_profile_retrieved"));
-				assertEquals(true, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicWithUserDeclaredLocationAsDerived() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-		DeviceProfile deviceProfile = new DeviceProfile
-				("IN", "India", "KA", "Karnataka",
-						"Bangalore", "Banglore-Custom", "Karnatak-Custom",
-						"KA-Custom", new HashMap<>(), 0L,"Bangalore",
-						"Karnataka");
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile);
-
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				assertEquals("3.0", outputEvent.get("ver"));
-				Map<String, Object> devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType);
-
-				Map<String, Object> deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType);
-				assertTrue(deviceSpec.isEmpty());
-
-				// Map<String, Object> uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType);
-				// assertEquals(true, uaSpec.isEmpty());
-
-				Map<String, Object> userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType);
-				assertEquals("Karnataka", userDeclared.get("state"));
-				assertEquals("Bangalore", userDeclared.get("district"));
-
-				Map<String, Object> derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType);
-				assertEquals("Karnataka", derivedLocationData.get("state"));
-				assertEquals("Bangalore", derivedLocationData.get("district"));
-				assertEquals("user-declared", derivedLocationData.get("from"));
-
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(true, flags.get("device_profile_retrieved"));
-				assertEquals(true, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicWithIpLocationAsDerived() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-		DeviceProfile deviceProfile = new DeviceProfile
-				("IN", "India", "KA", "Karnataka",
-						"Bangalore", "Banglore-Custom", "Karnatak-Custom",
-						"KA-Custom", new HashMap<>(), 0L,"",
-						"");
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile);
-
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				assertEquals("3.0", outputEvent.get("ver"));
-				Map<String, Object> devicedata = new Gson().fromJson(new Gson().toJson(outputEvent.get("devicedata")), mapType);
-
-				Map<String, Object> deviceSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("devicespec")), mapType);
-				assertTrue(deviceSpec.isEmpty());
-
-				// Map<String, Object> uaSpec = new Gson().fromJson(new Gson().toJson(devicedata.get("uaspec")), mapType);
-				// assertEquals(true, uaSpec.isEmpty());
-
-				Map<String, Object> userDeclared = new Gson().fromJson(new Gson().toJson(devicedata.get("userdeclared")), mapType);
-				assertEquals("", userDeclared.get("state"));
-				assertEquals("", userDeclared.get("district"));
-
-				Map<String, Object> derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType);
-				assertEquals("Karnataka", derivedLocationData.get("state"));
-				assertEquals("Banglore-Custom", derivedLocationData.get("district"));
-				assertEquals("ip-resolved", derivedLocationData.get("from"));
-
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(true, flags.get("device_profile_retrieved"));
-				assertEquals(true, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicWithOutDeviceProfile() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-		DeviceProfile deviceProfile = null;
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile);
-
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(false, flags.get("device_profile_retrieved"));
-				assertEquals(false, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventsToSuccessTopicWithOutDeviceProfileAndWithUserProfileLocationAsDerived() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INTERACT_EVENT);
-		DeviceProfile deviceProfile = null;
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("68dfc64a7751ad47617ac1a4e0531fb761ebea6f")).toReturn(deviceProfile);
-
-		jedisMock.set("393407b1-66b1-4c86-9080-b2bce9842886","{\"grade\":[4,5],\"district\":\"Bengaluru\",\"type\":\"Registered\",\"state\":\"Karnataka\"}");
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-
-				Map<String, Object> derivedLocationData = new Gson().fromJson(new Gson().toJson(outputEvent.get("derivedlocationdata")), mapType);
-				assertEquals("Karnataka", derivedLocationData.get("state"));
-				assertEquals("Bengaluru", derivedLocationData.get("district"));
-				assertEquals("user-profile", derivedLocationData.get("from"));
-
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(false, flags.get("device_profile_retrieved"));
-				assertEquals(true, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void deviceProfileFromMapShouldNotFailForEmptyValues() {
-		Map<String, String> deviceProfileMap = new HashMap<>();
-		deviceProfileMap.put("user_declared_state", "Tamil Nadu");
-		deviceProfileMap.put("user_declared_district", "Tiruchirappalli");
-
-		DeviceProfile deviceProfile = new DeviceProfile().fromMap(deviceProfileMap);
-		assertEquals("Tamil Nadu", deviceProfile.getUserDeclaredState());
-		assertEquals("Tiruchirappalli", deviceProfile.getUserDeclaredDistrict());
-
-		assertEquals(0, deviceProfile.getDevicespec().size());
-		assertTrue(StringUtils.isEmpty(deviceProfile.getState()));
-		assertTrue(StringUtils.isEmpty(deviceProfile.getCity()));
-		assertTrue(StringUtils.isEmpty(deviceProfile.getDistrictCustom()));
-		assertTrue(StringUtils.isEmpty(deviceProfile.getstateCodeCustom()));
-		assertTrue(StringUtils.isEmpty(deviceProfile.getstateCustomName()));
-		assertEquals(0L, deviceProfile.getFirstaccess().longValue());
-
-		assertFalse(deviceProfile.isLocationResolved());
-		assertFalse(deviceProfile.isDeviceProfileResolved());
-	}
-
-	@Test
-	public void shouldNotFailIfUserIdIsMissing() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.DEVICE_SUMMARY);
-		DeviceProfile deviceProfile = null;
-		stub(deviceProfileCacheMock.getDeviceProfileForDeviceId("099988ce86c4dbb9a4057ff611d38427")).toReturn(deviceProfile);
-
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() { }.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				Map<String, Object> flags = new Gson().fromJson(outputEvent.get("flags").toString(), mapType);
-				assertEquals(false, flags.get("device_profile_retrieved"));
-				assertEquals(false, flags.get("derived_location_retrieved"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT);
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING);
-		telemetryLocationUpdaterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-		return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-				assertEquals("kafka", systemStream.getSystem());
-				assertEquals(stream, systemStream.getStream());
-				return true;
-			}
-		};
-	}
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java b/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java
deleted file mode 100644
index 87db5b0d83..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/java/org/ekstep/ep/samza/util/DeviceProfileCacheTest.java
+++ /dev/null
@@ -1,38 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import org.ekstep.ep.samza.domain.DeviceProfile;
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.stub;
-
-public class DeviceProfileCacheTest {
-
-
-    private DeviceProfileCache cache;
-
-    @Test
-    public void shouldReturnNullWhenDeviceIdIsNotPresent() {
-        cache = mock(DeviceProfileCache.class);
-        stub(cache.getDeviceProfileFromCache("test")).toReturn(null);
-        Assert.assertEquals(cache.getDeviceProfileFromCache("564783-5439"), null);
-    }
-
-    @Test
-    public void shouldReturnProfileInfo() {
-        DeviceProfile profile = new DeviceProfile("IN", "India", "KA", "Karnataka",
-                "Bengaluru");
-        cache = mock(DeviceProfileCache.class);
-        stub(cache.getDeviceProfileFromCache("37654387-543534")).toReturn(profile);
-        DeviceProfile res = cache.getDeviceProfileFromCache("37654387-543534");
-        Assert.assertEquals(res.getCountryCode(), "IN");
-        Assert.assertEquals(res.getCountry(), "India");
-        Assert.assertEquals(res.getStateCode(), "KA");
-        Assert.assertEquals(res.getState(), "Karnataka");
-        Assert.assertEquals(res.getCity(), "Bengaluru");
-
-    }
-
-
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json
deleted file mode 100755
index 4c7261b025..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json
deleted file mode 100755
index 9ac4103bd6..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/common_empty_sid_uid.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "pdata": {
-      "$ref": "file:src/test/resources/2.2/pdata.json"
-    },
-    "gdata": {
-      "$ref": "file:src/test/resources/2.2/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "file:src/test/resources/2.2/etags.json"
-    },
-    "cdata": {
-      "$ref": "file:src/test/resources/2.2/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/2.2/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json
deleted file mode 100755
index e00c02e041..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/etags.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "type": "object",
-    "properties": {
-      "app": {
-        "id": "http://api.ekstep.org/telemetry/etags/app",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/app/items",
-            "type": "string"
-        }
-      },
-      "partner": {
-        "id": "http://api.ekstep.org/telemetry/etags/partner",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/partner/items",
-            "type": "string"
-        }
-      },
-      "dims": {
-        "id": "http://api.ekstep.org/telemetry/etags/dims",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/dims/items",
-            "type": "string"
-        }
-      }
-    },
-    "additionalProperties": false
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json
deleted file mode 100755
index 1b603a1b7b..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json
deleted file mode 100755
index 14350be425..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/ge_error.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "file:src/test/resources/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_ERROR"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "id",
-                                "err",
-                                "eventId",
-                                "data"
-                            ],
-                            "properties": {
-                                "err": {
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "SYSTEM",
-                                        "GENIE",
-                                        "GAME",
-                                        "GENIESERVICES"
-                                    ]
-                                },
-                                "id": {
-                                    "type": "string"
-                                },
-                                "eventId": {
-                                    "type": "string"
-                                },
-                                "data": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
diff --git a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json b/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json
deleted file mode 100755
index cd509bc58b..0000000000
--- a/data-pipeline/telemetry-location-updater/src/test/resources/2.2/pdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/pdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/pdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/data-pipeline/telemetry-redacter/pom.xml b/data-pipeline/telemetry-redacter/pom.xml
deleted file mode 100644
index 69a1ac02f7..0000000000
--- a/data-pipeline/telemetry-redacter/pom.xml
+++ /dev/null
@@ -1,190 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>telemetry-redacter</artifactId>
-    <version>0.0.1</version>
-    <packaging>jar</packaging>
-    <name>TelemetryRedacter</name>
-    <description>
-        Telemetry redacter to redact user input values
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-         <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-		    <groupId>it.ozimov</groupId>
-		    <artifactId>embedded-redis</artifactId>
-		    <version>0.7.1</version>
-		    <scope>test</scope>
-		</dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/telemetry-redacter/src/main/assembly/src.xml b/data-pipeline/telemetry-redacter/src/main/assembly/src.xml
deleted file mode 100644
index d3ea2ee0e2..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/telemetry-redacter.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:telemetry-redacter</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties b/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties
deleted file mode 100644
index 81fe7fda24..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/config/telemetry-redacter.properties
+++ /dev/null
@@ -1,85 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.TelemetryRedacter
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_redacter_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.TelemetryRedacterTask
-task.inputs=kafka.__env__.telemetry.assess.redact
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-redacted.route.topic=__env__.telemetry.raw
-nonredacted.route.topic=__env__.telemetry.assess.raw
-failed.topic.name=__env__.telemetry.failed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,assess-route-success-count,failed-message-count,skipped-message-count,cache-miss-count,cache-hit-count
-
-# redis
-# redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=2
-redis.connection.idle.max=2
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-
-# redis database index
-redis.contentDB.index=5
\ No newline at end of file
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index 7361424e13..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,42 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-
-public class Event extends Events {
-
-  public Event(Map<String, Object> map) {
-    super(map);
-  }
-
-  public void clearUserInput() {
-    if ("ASSESS".equals(eid())) {
-      telemetry.add("edata.resvalues", new ArrayList<>());
-    } else {
-      telemetry.add("edata.values", new ArrayList<>());
-    }
-  }
-
-  public String questionId() {
-    if ("ASSESS".equals(eid())) {
-      NullableValue<String> itemId = telemetry.<String>read("edata.item.id");
-      return itemId.isNull() ? null : itemId.value();
-    } else {
-      NullableValue<String> itemId = telemetry.<String>read("edata.target.id");
-      return itemId.isNull() ? null : itemId.value();
-    }
-  }
-
-  public List<Object> readResValues() {
-    if ("ASSESS".equals(eid())) {
-      return telemetry.<List<Object>>read("edata.resvalues").value();
-    } else {
-      return telemetry.<List<Object>>read("edata.values").value();
-    }
-  }
-
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java
deleted file mode 100644
index 2ffa144691..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/service/TelemetryRedacterService.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import static java.text.MessageFormat.format;
-
-import java.util.Map;
-
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.TelemetryRedacterSink;
-import org.ekstep.ep.samza.task.TelemetryRedacterSource;
-import org.ekstep.ep.samza.util.QuestionDataCache;
-
-public class TelemetryRedacterService {
-
-  private static Logger LOGGER = new Logger(TelemetryRedacterService.class);
-  private QuestionDataCache cache = null;
-  private JobMetrics metrics = null;
-
-  public TelemetryRedacterService(QuestionDataCache cache, JobMetrics metrics) {
-    this.cache = cache;
-    this.metrics = metrics;
-  }
-
-  public void process(TelemetryRedacterSource source, TelemetryRedacterSink sink) {
-    Event event = null;
-    try {
-      boolean redact = false;
-      event = source.getEvent();
-      String questionId = event.questionId();
-      if (null == questionId) {
-        metrics.incSkippedCounter();
-      } else {
-        Map<String, Object> questionData = cache.getData(event.questionId());
-        if (null == questionData) {
-          metrics.incCacheMissCounter();
-        } else {
-          metrics.incCacheHitCounter();
-          if (questionData.containsKey("questionType")
-              && "Registration".equalsIgnoreCase((String) questionData.get("questionType"))) {
-            redact = true;
-          }
-        }
-      }
-      if (redact) {
-        sink.toNonRedactedRoute(event);
-        event.clearUserInput();
-      }
-      sink.toRedactedRoute(event);
-    } catch (Exception e) {
-      LOGGER.error(null,
-          format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:", event),
-          e);
-      sink.toErrorTopic(source.getMessage());
-    }
-  }
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java
deleted file mode 100644
index 48bc873baa..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterConfig.java
+++ /dev/null
@@ -1,35 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-public class TelemetryRedacterConfig {
-
-  private final String JOB_NAME = "TelemetryRedacter";
-
-  private String redactedRouteTopic;
-  private String nonRedactedRouteTopic;
-  private String failedTopic;
-
-  public TelemetryRedacterConfig(Config config) {
-    redactedRouteTopic = config.get("redacted.route.topic", "telemetry.raw");
-    failedTopic = config.get("failed.topic.name", "telemetry.failed");
-    nonRedactedRouteTopic = config.get("nonredacted.route.topic", "telemetry.assess.raw");
-  }
-
-  public String getRedactedRouteTopic() {
-    return redactedRouteTopic;
-  }
-
-  public String getNonRedactedRouteTopic() {
-    return nonRedactedRouteTopic;
-  }
-
-  public String jobName() {
-    return JOB_NAME;
-  }
-
-  public String failedTopic() {
-    return failedTopic;
-  }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java
deleted file mode 100644
index dbdce6ddbb..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSink.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class TelemetryRedacterSink extends BaseSink {
-
-  private TelemetryRedacterConfig config;
-
-  public TelemetryRedacterSink(MessageCollector collector, JobMetrics metrics, TelemetryRedacterConfig config) {
-    super(collector, metrics);
-    this.config = config;
-  }
-
-  public void toRedactedRoute(Event event) {
-    toTopic(config.getRedactedRouteTopic(), event.did(), event.getJson());
-    metrics.incSuccessCounter();
-  }
-  
-  public void toNonRedactedRoute(Event event) {
-    toTopic(config.getNonRedactedRouteTopic(), event.did(), event.getJson());
-    metrics.incAssessRouteSuccessCounter();
-  }
-
-  public void toErrorTopic(String message) {
-    toTopic(config.failedTopic(), null, message);
-    metrics.incFailedCounter();
-  }
-
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java
deleted file mode 100644
index d2912c191e..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class TelemetryRedacterSource {
-  
-    static Logger LOGGER = new Logger(TelemetryRedacterSource.class);
-
-    private IncomingMessageEnvelope envelope;
-
-    public TelemetryRedacterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java
deleted file mode 100644
index 2dfa05711c..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/task/TelemetryRedacterTask.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.service.TelemetryRedacterService;
-import org.ekstep.ep.samza.util.QuestionDataCache;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class TelemetryRedacterTask extends BaseSamzaTask {
-
-	static Logger LOGGER = new Logger(TelemetryRedacterTask.class);
-	private TelemetryRedacterConfig config;
-	private JobMetrics metrics;
-	private TelemetryRedacterService service;
-	private RedisConnect redisConnect;
-
-	public TelemetryRedacterTask(Config config, TaskContext context) {
-		init(config, context);
-	}
-
-	public TelemetryRedacterTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		
-	  this.redisConnect = new RedisConnect(config);
-		this.config = new TelemetryRedacterConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		service = new TelemetryRedacterService(new QuestionDataCache(config, redisConnect), metrics);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator)
-			throws Exception {
-
-		TelemetryRedacterSink sink = new TelemetryRedacterSink(collector, metrics, config);
-		TelemetryRedacterSource source = new TelemetryRedacterSource(envelope);
-		service.process(source, sink);
-	}
-
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java b/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java
deleted file mode 100644
index 21d56e7327..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/java/org/ekstep/ep/samza/util/QuestionDataCache.java
+++ /dev/null
@@ -1,57 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import java.lang.reflect.Type;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.samza.config.Config;
-import org.ekstep.ep.samza.core.Logger;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-
-import redis.clients.jedis.Jedis;
-import redis.clients.jedis.exceptions.JedisException;
-
-public class QuestionDataCache {
-
-  private static Logger LOGGER = new Logger(QuestionDataCache.class);
-
-  private RedisConnect redisPool;
-  private Jedis redisConnection;
-  @SuppressWarnings("serial")
-  private Type mapType = new TypeToken<Map<String, Object>>() {
-  }.getType();
-  private Gson gson = new Gson();
-  private int databaseIndex;
-
-  public QuestionDataCache(Config config, RedisConnect redisConnect) {
-
-    this.databaseIndex = config.getInt("redis.contentDB.index", 5);
-    this.redisPool = redisConnect;
-    this.redisConnection = this.redisPool.getConnection(databaseIndex);
-  }
-
-  private Map<String, Object> getDataFromCache(String key) {
-    Map<String, Object> cacheData = new HashMap<>();
-    String data = redisConnection.get(key);
-    if (data != null && !data.isEmpty()) {
-      cacheData = gson.fromJson(data, mapType);
-    }
-    return cacheData;
-  }
-
-  public Map<String, Object> getData(String key) {
-    Map<String, Object> cacheDataMap;
-    try {
-      cacheDataMap = getDataFromCache(key);
-    } catch (JedisException ex) {
-      LOGGER.error("", "Exception when retrieving data from redis cache ", ex);
-      this.redisConnection.close();
-      this.redisConnection.close();
-      this.redisConnection = redisPool.getConnection(databaseIndex);
-      cacheDataMap = getDataFromCache(key);
-    }
-    return cacheDataMap;
-  }
-}
diff --git a/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml b/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/telemetry-redacter/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index 16618ba5f3..0000000000
--- a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,208 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-
-import java.util.Map;
-
-public class EventFixture {
-
-  public static final String UNPARSABLE_GE_GENIE_UPDATE_EVENT = "{\n"
-      + "  \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n"
-      + "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" + "  \"edata\": {\n" + "    \"eks\": {\n"
-      + "      \"mode\": \"WIFI\",\n" + "      \"ver\": \"12\",\n" + "      \"size\": 12.67,\n"
-      + "      \"err\": \"\",\n" + "      \"referrer\": [\n" + "        {\n" + "          \"action\": \"INSTALL\",\n"
-      + "          \"utmsource\": \"Ekstep\",\n" + "          \"utmmedium\": \"Portal\",\n"
-      + "          \"utmterm\": \"December 2016\",\n" + "          \"utmcontent\": \"Ramayana\",\n"
-      + "          \"utmcampaign\": \"Epics of India\"\n" + "        }\n" + "      ]\n" + "    }\n" + "  },\n"
-      + "  \"eid\": \"GE_GENIE_UPDATE\",\n" + "  \"gdata\": {\n" + "    \"id\": \"genie.android\",\n"
-      + "    \"ver\": \"1.0\"\n" + "  },\n" + "  \"sid\": \"\",\n" + "  \"ets\": 1454064092546,\n"
-      + "  \"uid\": \"\",\n" + "  \"ver\": \"2.0\",\n" + "  \"cdata\": [\n" + "    {\n"
-      + "      \"id\": \"correlationid\",\n" + "      \"type\": \"correlationtype\"\n" + "    ";
-  
-  public static final String RESPONSE_EVENT = "{\"eid\":\"RESPONSE\",\"ets\":1586759864200,\"ver\":\"3.0\",\"mid\":\"RESPONSE:17d97e7cba61bd7c996d3ae71b9706ef\",\"actor\":{\"id\":\"c5b5732b-b8d6-44ad-bf52-d8f7fc513ac3\",\"type\":\"User\"},\"context\":{\"channel\":\"0126825293972439041\",\"pdata\":{\"id\":\"preprod.diksha.portal\",\"ver\":\"2.8.7\",\"pid\":\"sunbird-portal.contentplayer\"},\"env\":\"contentplayer\",\"sid\":\"zSizgfO7mzdZ--MhlGCzszgdNhKW6MCp\",\"did\":\"a915339fbca0afddbf537a1792f01791\",\"cdata\":[{\"id\":\"28d1f55313c9c14df8ea86ca1dc73a92\",\"type\":\"ContentSession\"},{\"id\":\"fed7932935f2223870beb120bf58aeed\",\"type\":\"PlaySession\"}],\"rollup\":{\"l1\":\"0126825293972439041\"}},\"object\":{\"id\":\"do_2129946748092334081914\",\"type\":\"Content\",\"ver\":\"2\",\"rollup\":{}},\"tags\":[\"0126825293972439041\"],\"edata\":{\"target\":{\"id\":\"do_21299463432309145612386\",\"ver\":\"1.0\",\"type\":\"AssessmentItem\"},\"type\":\"CHOOSE\",\"values\":[{\"option0\":\"text\"}]}}";
-  
-  public static final String ASSESS_EVENT = "{" +
-      "      \"eid\": \"ASSESS\",\n" +
-      "      \"ets\": 1568891738245,\n" +
-      "      \"ver\": \"3.1\",\n" +
-      "      \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" +
-      "      \"actor\": {\n" +
-      "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-      "        \"type\": \"User\"\n" +
-      "      },\n" +
-      "      \"context\": {\n" +
-      "        \"channel\": \"0124784842112040965\",\n" +
-      "        \"pdata\": {\n" +
-      "          \"id\": \"staging.diksha.portal\",\n" +
-      "          \"ver\": \"2.4.0\",\n" +
-      "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-      "        },\n" +
-      "        \"env\": \"contentplayer\",\n" +
-      "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-      "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-      "        \"cdata\": [\n" +
-      "          {\n" +
-      "            \"id\": \"do_2128415652377067521127\",\n" +
-      "            \"type\": \"course\"\n" +
-      "          },\n" +
-      "          {\n" +
-      "            \"type\": \"batch\",\n" +
-      "            \"id\": \"012846671379595264119\"\n" +
-      "          },\n" +
-      "          {\n" +
-      "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-      "            \"type\": \"ContentSession\"\n" +
-      "          }\n" +
-      "        ],\n" +
-      "        \"rollup\": {\n" +
-      "          \"l1\": \"0124784842112040965\"\n" +
-      "        }\n" +
-      "      },\n" +
-      "      \"object\": {\n" +
-      "        \"id\": \"do_212686723743318016173\",\n" +
-      "        \"type\": \"Content\",\n" +
-      "        \"ver\": \"1\",\n" +
-      "        \"rollup\": {\n" +
-      "          \"l1\": \"do_2128415652377067521127\",\n" +
-      "          \"l2\": \"do_2128415660716359681128\"\n" +
-      "        }\n" +
-      "      },\n" +
-      "      \"tags\": [\n" +
-      "        \"0124784842112040965\"\n" +
-      "      ],\n" +
-      "      \"edata\": {\n" +
-      "        \"item\": {\n" +
-      "          \"id\": \"801ae93c-8807-4be5-8853-dd49362d8776\",\n" +
-      "          \"maxscore\": 1,\n" +
-      "          \"type\": \"mcq\",\n" +
-      "          \"exlength\": 0,\n" +
-      "          \"params\": [\n" +
-      "            {\n" +
-      "              \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" +
-      "            }\n" +
-      "          ],\n" +
-      "          \"uri\": \"\",\n" +
-      "          \"title\": \"What is the Full form of WHO..?\\n\",\n" +
-      "          \"mmc\": [],\n" +
-      "          \"mc\": [],\n" +
-      "          \"desc\": \"\"\n" +
-      "        },\n" +
-      "        \"index\": 1,\n" +
-      "        \"pass\": \"No\",\n" +
-      "        \"score\": 0,\n" +
-      "        \"resvalues\": [\n" +
-      "          {\n" +
-      "            \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-      "          }\n" +
-      "        ],\n" +
-      "        \"duration\": 4\n" +
-      "      }\n" +
-      "    }";
-  
-  public static final String ASSESS_EVENT_WITHOUT_QID = "{" +
-      "      \"eid\": \"ASSESS\",\n" +
-      "      \"ets\": 1568891738245,\n" +
-      "      \"ver\": \"3.1\",\n" +
-      "      \"mid\": \"ASSESS:135815023ec32a430632ba5d7f84fe18\",\n" +
-      "      \"actor\": {\n" +
-      "        \"id\": \"ff1c4bdf-27e2-49bc-a53f-6e304bb3a87f\",\n" +
-      "        \"type\": \"User\"\n" +
-      "      },\n" +
-      "      \"context\": {\n" +
-      "        \"channel\": \"0124784842112040965\",\n" +
-      "        \"pdata\": {\n" +
-      "          \"id\": \"staging.diksha.portal\",\n" +
-      "          \"ver\": \"2.4.0\",\n" +
-      "          \"pid\": \"sunbird-portal.contentplayer\"\n" +
-      "        },\n" +
-      "        \"env\": \"contentplayer\",\n" +
-      "        \"sid\": \"wqmQpaYc9mRD6jdU6NOWuBTEyGMPXFEe\",\n" +
-      "        \"did\": \"a08946e8b72abfeeff6642f245d470cb\",\n" +
-      "        \"cdata\": [\n" +
-      "          {\n" +
-      "            \"id\": \"do_2128415652377067521127\",\n" +
-      "            \"type\": \"course\"\n" +
-      "          },\n" +
-      "          {\n" +
-      "            \"type\": \"batch\",\n" +
-      "            \"id\": \"012846671379595264119\"\n" +
-      "          },\n" +
-      "          {\n" +
-      "            \"id\": \"f3ec2acf4360e93172b9234e29e38be4\",\n" +
-      "            \"type\": \"ContentSession\"\n" +
-      "          }\n" +
-      "        ],\n" +
-      "        \"rollup\": {\n" +
-      "          \"l1\": \"0124784842112040965\"\n" +
-      "        }\n" +
-      "      },\n" +
-      "      \"object\": {\n" +
-      "        \"id\": \"do_212686723743318016173\",\n" +
-      "        \"type\": \"Content\",\n" +
-      "        \"ver\": \"1\",\n" +
-      "        \"rollup\": {\n" +
-      "          \"l1\": \"do_2128415652377067521127\",\n" +
-      "          \"l2\": \"do_2128415660716359681128\"\n" +
-      "        }\n" +
-      "      },\n" +
-      "      \"tags\": [\n" +
-      "        \"0124784842112040965\"\n" +
-      "      ],\n" +
-      "      \"edata\": {\n" +
-      "        \"item\": {\n" +
-      "          \"maxscore\": 1,\n" +
-      "          \"type\": \"mcq\",\n" +
-      "          \"exlength\": 0,\n" +
-      "          \"params\": [\n" +
-      "            {\n" +
-      "              \"1\": \"{\\\"text\\\":\\\"World Health Organizaton\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"3\": \"{\\\"text\\\":\\\"Work hell Organization\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"4\": \"{\\\"text\\\":\\\"None of The above\\\\n\\\"}\"\n" +
-      "            },\n" +
-      "            {\n" +
-      "              \"answer\": \"{\\\"correct\\\":[\\\"1\\\"]}\"\n" +
-      "            }\n" +
-      "          ],\n" +
-      "          \"uri\": \"\",\n" +
-      "          \"title\": \"What is the Full form of WHO..?\\n\",\n" +
-      "          \"mmc\": [],\n" +
-      "          \"mc\": [],\n" +
-      "          \"desc\": \"\"\n" +
-      "        },\n" +
-      "        \"index\": 1,\n" +
-      "        \"pass\": \"No\",\n" +
-      "        \"score\": 0,\n" +
-      "        \"resvalues\": [\n" +
-      "          {\n" +
-      "            \"2\": \"{\\\"text\\\":\\\"Work Heavy Organization\\\\n\\\"}\"\n" +
-      "          }\n" +
-      "        ],\n" +
-      "        \"duration\": 4\n" +
-      "      }\n" +
-      "    }"; 
-      
-
-  public static Map<String, Object> getMap(String message) {
-    return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-  }
-
-}
diff --git a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java b/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java
deleted file mode 100644
index ca8a37a370..0000000000
--- a/data-pipeline/telemetry-redacter/src/test/java/org/ekstep/ep/samza/task/TelemetryRedacterTaskTest.java
+++ /dev/null
@@ -1,196 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.stub;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.Map;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import com.google.gson.Gson;
-
-import redis.clients.jedis.Jedis;
-import redis.embedded.RedisServer;
-
-public class TelemetryRedacterTaskTest {
-
-  private MessageCollector collectorMock;
-  private TaskContext contextMock;
-  private MetricsRegistry metricsRegistry;
-  private Counter counter;
-  private TaskCoordinator coordinatorMock;
-  private IncomingMessageEnvelope envelopeMock;
-  private Config configMock;
-  private TelemetryRedacterTask telemetryRedacterTask;
-  private RedisServer redisServer;
-  private RedisConnect redisConnect;
-
-  @Before
-  public void setUp() throws IOException {
-
-    redisServer = new RedisServer(6379);
-    redisServer.start();
-    collectorMock = mock(MessageCollector.class);
-    contextMock = Mockito.mock(TaskContext.class);
-    metricsRegistry = Mockito.mock(MetricsRegistry.class);
-    counter = Mockito.mock(Counter.class);
-    coordinatorMock = mock(TaskCoordinator.class);
-    envelopeMock = mock(IncomingMessageEnvelope.class);
-    configMock = Mockito.mock(Config.class);
-    when(configMock.get("redis.host", "localhost")).thenReturn("localhost");
-    when(configMock.getInt("redis.port", 6379)).thenReturn(6379);
-    when(configMock.getInt("redis.connection.max", 2)).thenReturn(2);
-    when(configMock.getInt("redis.connection.idle.max", 2)).thenReturn(2);
-    when(configMock.getInt("redis.connection.idle.min", 1)).thenReturn(1);
-    when(configMock.getInt("redis.connection.minEvictableIdleTimeSeconds", 120)).thenReturn(120);
-    when(configMock.getInt("redis.connection.timeBetweenEvictionRunsSeconds", 300)).thenReturn(300);
-    when(configMock.getInt("redis.contentDB.index", 5)).thenReturn(5);
-    redisConnect = new RedisConnect(configMock);
-
-    stub(configMock.get("redacted.route.topic", "telemetry.raw")).toReturn("telemetry.raw");
-    stub(configMock.get("nonredacted.route.topic", "telemetry.assess.raw")).toReturn("telemetry.assess.raw");
-    stub(configMock.get("failed.topic.name", "telemetry.failed")).toReturn("telemetry.failed");
-
-    stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-    stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-
-    telemetryRedacterTask = new TelemetryRedacterTask(configMock, contextMock);
-  }
-
-  @After
-  public void tearDown() {
-    redisServer.stop();
-  }
-
-  @Test
-  public void shouldSendEventToErroTopicIfEventIsNotParseable() throws Exception {
-
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_GE_GENIE_UPDATE_EVENT);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock).send(argThat(validateOutputTopic("telemetry.failed")));
-  }
-
-  @Test
-  public void shouldRedactEventAndSendToRawTopic() throws Exception {
-
-    Jedis jedis = redisConnect.getConnection(5);
-    jedis.set("801ae93c-8807-4be5-8853-dd49362d8776", "{\"questionType\":\"Registration\"}");
-    jedis.close();
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock, times(2)).send(argThat(validateRedactedTopic(true)));
-  }
-  
-  @Test
-  public void shouldRedactEventAndSendToRawTopic2() throws Exception {
-
-    Jedis jedis = redisConnect.getConnection(5);
-    jedis.set("do_21299463432309145612386", "{\"questionType\":\"Registration\"}");
-    jedis.close();
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.RESPONSE_EVENT);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock, times(2)).send(argThat(validateRedactedTopic(true)));
-  }
-
-  @Test
-  public void shouldNotRedactEventAndSendToRawTopic1() throws Exception {
-    
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT_WITHOUT_QID);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock, times(1)).send(argThat(validateOutputTopic("telemetry.raw")));
-  }
-  
-  @Test
-  public void shouldNotRedactEventAndSendToRawTopic2() throws Exception {
-    
-    Jedis jedis = redisConnect.getConnection(5);
-    jedis.del("801ae93c-8807-4be5-8853-dd49362d8776");
-    jedis.close();
-
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock, times(1)).send(argThat(validateOutputTopic("telemetry.raw")));
-  }
-  
-  @Test
-  public void shouldNotRedactEventAndSendToRawTopic3() throws Exception {
-
-    Jedis jedis = redisConnect.getConnection(5);
-    jedis.set("801ae93c-8807-4be5-8853-dd49362d8776", "{\"questionType\":\"Survey\"}");
-    jedis.close();
-    stub(envelopeMock.getMessage()).toReturn(EventFixture.ASSESS_EVENT);
-    telemetryRedacterTask.process(envelopeMock, collectorMock, coordinatorMock);
-    verify(collectorMock).send(argThat(validateOutputTopic("telemetry.raw")));
-  }
-  
-  public ArgumentMatcher<OutgoingMessageEnvelope> validateRedactedTopic(final boolean redacted) {
-    return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-      @Override
-      public boolean matches(Object o) {
-        OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-        SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-        assertEquals("kafka", systemStream.getSystem());
-        assertTrue(Arrays.asList("telemetry.raw", "telemetry.assess.raw").contains(systemStream.getStream()));
-
-        if ("telemetry.raw".equals(systemStream.getStream())) {
-          String message = (String) outgoingMessageEnvelope.getMessage();
-          Map<String, Object> eventMap = new Gson().fromJson(message, Map.class);
-          Event event = new Event(eventMap);
-          if (redacted) {
-            assertEquals(0, event.readResValues().size());
-          }
-        }
-        if ("telemetry.assess.raw".equals(systemStream.getStream())) {
-          System.out.println("Redacted and message sent to raw stream");
-          String message = (String) outgoingMessageEnvelope.getMessage();
-          Map<String, Object> eventMap = new Gson().fromJson(message, Map.class);
-          Event event = new Event(eventMap);
-          assertEquals(1, event.readResValues().size());
-        }
-
-        return true;
-      }
-    };
-  }
-
-  public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final String stream) {
-    return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-      @Override
-      public boolean matches(Object o) {
-        OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-        SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-        assertEquals("kafka", systemStream.getSystem());
-        assertEquals(stream, systemStream.getStream());
-
-        return true;
-      }
-    };
-  }
-
-}
diff --git a/data-pipeline/telemetry-router/pom.xml b/data-pipeline/telemetry-router/pom.xml
deleted file mode 100644
index 0a2bcf01dc..0000000000
--- a/data-pipeline/telemetry-router/pom.xml
+++ /dev/null
@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>telemetry-router</artifactId>
-        <version>0.1.6</version>
-    <packaging>jar</packaging>
-    <name>TelemetryRouter</name>
-    <description>
-        Telemetry router based on event priority
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-         <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/telemetry-router/src/main/assembly/src.xml b/data-pipeline/telemetry-router/src/main/assembly/src.xml
deleted file mode 100644
index 3dd4a548b0..0000000000
--- a/data-pipeline/telemetry-router/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/telemetry-router.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-kv-rocksdb_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:telemetry-router</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties b/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties
deleted file mode 100644
index 7775b12bc2..0000000000
--- a/data-pipeline/telemetry-router/src/main/config/telemetry-router.properties
+++ /dev/null
@@ -1,79 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.TelemetryRouter
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_router_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.TelemetryRouterTask
-task.inputs=kafka.__env__.telemetry.unique,kafka.__env__.telemetry.derived.unique
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-systems.kafka.streams.__env__.telemetry.derived.unique.samza.priority=1
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-router.events.primary.route.topic=__env__.telemetry.sink
-router.events.secondary.route.events=__tr_secondary_route_events__
-router.events.secondary.route.topic=__env__.telemetry.log
-router.events.audit.route.topic=__env__.telemetry.audit
-router.events.share.route.topic=__env__.telemetry.share
-output.failed.topic.name=__env__.telemetry.failed
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-router.events.log.topic.name=__env__.events.log
-router.events.error.topic.name=__env__.events.error
-pipeline.metrics.list=error-message-count,primary-route-success-count,secondary-route-success-count,audit-route-success-count,log-route-success-count,error-route-success-count
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index c6a879ac00..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.task.TelemetryRouterConfig;
-
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public void markFailure(String error, TelemetryRouterConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tr_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        telemetry.add("metadata.tr_error", error);
-        telemetry.add("metadata.src", config.jobName());
-    }
-
-    public void setTimestamp() {
-        Double ets = safelyParse(path.ets());
-        SimpleDateFormat simple = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ");
-
-        if (ets != null) {
-            String updatedTs = simple.format(new Date(ets.longValue()));
-            telemetry.add(path.ts(), updatedTs);
-        } else {
-            telemetry.add(path.ts(), simple.format(new Date()));
-        }
-    }
-
-    private Double safelyParse(String etsField) {
-        try {
-            NullableValue<Double> time = telemetry.read(etsField);
-            return time.value();
-        } catch (ClassCastException e) {
-            NullableValue<Long> timeInLong = telemetry.read(etsField);
-            return Double.valueOf(timeInLong.value());
-        }
-    }
-
-}
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java
deleted file mode 100644
index 6d021465a3..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/service/TelemetryRouterService.java
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.TelemetryRouterConfig;
-import org.ekstep.ep.samza.task.TelemetryRouterSink;
-import org.ekstep.ep.samza.task.TelemetryRouterSource;
-
-import java.util.List;
-
-import static java.text.MessageFormat.format;
-
-public class TelemetryRouterService {
-
-	private static Logger LOGGER = new Logger(TelemetryRouterService.class);
-	private final TelemetryRouterConfig config;
-
-	public TelemetryRouterService(TelemetryRouterConfig config) {
-		this.config = config;
-	}
-
-	public void process(TelemetryRouterSource source, TelemetryRouterSink sink) {
-		Event event = null;
-		try {
-			event = source.getEvent();
-			String eid = event.eid();
-			List<String> secondaryRouteEvents = this.config.getSecondaryRouteEvents();
-			if (secondaryRouteEvents.contains(eid)) {
-				if("LOG".equalsIgnoreCase(eid)) { sink.toLogRoute(event); }
-				else if("ERROR".equalsIgnoreCase(eid)) { sink.toErrorRoute(event); }
-				else { sink.toSecondaryRoute(event); }
-			} else {
-				if ("SHARE".equalsIgnoreCase(eid)) {
-					sink.toShareEventRouter(event);
-				} else {
-					sink.toPrimaryRoute(event);
-				}
-			}
-			if ("AUDIT".equalsIgnoreCase(eid)) {
-				sink.toAuditRoute(event);
-			}
-		} catch (JsonSyntaxException e) {
-			LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-			sink.toMalformedTopic(source.getMessage());
-		} catch (Exception e) {
-			LOGGER.error(null,
-					format("EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:",
-							event),
-					e);
-			sink.toErrorTopic(event, e.getMessage());
-		}
-	}
-}
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java
deleted file mode 100644
index 209061f957..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterConfig.java
+++ /dev/null
@@ -1,79 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class TelemetryRouterConfig {
-
-    private final String JOB_NAME = "TelemetryRouter";
-
-    private String failedTopic;
-    private String primaryRouteTopic;
-    private String secondaryRouteTopic;
-    private String secondaryRouteEvents;
-    private String malformedTopic;
-    private String auditRouteTopic;
-    private String shareEventRouterTopic;
-    private String logRouteTopic;
-    private String errorRouteTopic;
-
-    public TelemetryRouterConfig(Config config) {
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        primaryRouteTopic = config.get("router.events.primary.route.topic", "telemetry.sink");
-        secondaryRouteEvents = config.get("router.events.secondary.route.events", "LOG,ERROR");
-        secondaryRouteTopic = config.get("router.events.secondary.route.topic", "telemetry.log");
-        auditRouteTopic = config.get("router.events.audit.route.topic", "telemetry.audit");
-        shareEventRouterTopic = config.get("router.events.share.route.topic", "telemetry.share");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        logRouteTopic = config.get("router.events.log.topic.name","events.log");
-        errorRouteTopic = config.get("router.events.error.topic.name","events.error");
-
-    }
-
-    public String getPrimaryRouteTopic() {
-        return primaryRouteTopic;
-    }
-
-
-    public String getSecondaryRouteTopic() {
-        return secondaryRouteTopic;
-    }
-
-    public String getLogRouteTopic() {
-        return logRouteTopic;
-    }
-
-    public String getErrorRouteTopic() {
-        return errorRouteTopic;
-    }
-
-
-    public List<String> getSecondaryRouteEvents() {
-        String[] events = this.secondaryRouteEvents.split(",");
-        return Arrays.asList(events);
-    }
-
-    public String getAuditRouteTopic() {
-        return auditRouteTopic;
-    }
-
-    public String getShareEventRouterTopic() {
-        return shareEventRouterTopic;
-    }
-
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java
deleted file mode 100644
index 7934d1412a..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSink.java
+++ /dev/null
@@ -1,59 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class TelemetryRouterSink extends BaseSink {
-
-	private TelemetryRouterConfig config;
-
-	public TelemetryRouterSink(MessageCollector collector, JobMetrics metrics, TelemetryRouterConfig config) {
-		super(collector, metrics);
-		this.config = config;
-	}
-
-	public void toPrimaryRoute(Event event) {
-		event.setTimestamp();
-		toTopic(config.getPrimaryRouteTopic(), event.did(), event.getJson());
-		metrics.incPrimaryRouteSuccessCounter();
-	}
-
-	public void toErrorTopic(Event event, String errorMessage) {
-		event.markFailure(errorMessage, config);
-		toTopic(config.failedTopic(), event.mid(), event.getJson());
-		metrics.incErrorCounter();
-	}
-
-	public void toMalformedTopic(String message) {
-		toTopic(config.malformedTopic(), null, message);
-		metrics.incErrorCounter();
-	}
-
-	public void toSecondaryRoute(Event event) {
-		toTopic(config.getSecondaryRouteTopic(), event.did(), event.getJson());
-		metrics.incSecondaryRouteSuccessCounter();
-	}
-
-	public void toLogRoute(Event event) {
-		toTopic(config.getLogRouteTopic(), event.did(), event.getJson());
-		metrics.incLogRouteSuccessCounter();
-	}
-
-	public void toErrorRoute(Event event) {
-		toTopic(config.getErrorRouteTopic(), event.did(), event.getJson());
-		metrics.incErrorRouteSuccessCounter();
-	}
-
-	public void toAuditRoute(Event event) {
-		toTopic(config.getAuditRouteTopic(), event.mid(), event.getJson());
-		metrics.incAuditRouteSuccessCounter();
-	}
-
-	public void toShareEventRouter(Event event) {
-		toTopic(config.getShareEventRouterTopic(), event.mid(), event.getJson());
-		metrics.incShareEventRouteSuccessCounter();
-	}
-
-}
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java
deleted file mode 100644
index 48f0a98eb1..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class TelemetryRouterSource {
-    static Logger LOGGER = new Logger(TelemetryRouterSource.class);
-
-    private IncomingMessageEnvelope envelope;
-
-    public TelemetryRouterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-
-}
diff --git a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java b/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java
deleted file mode 100644
index da28eee5c1..0000000000
--- a/data-pipeline/telemetry-router/src/main/java/org/ekstep/ep/samza/task/TelemetryRouterTask.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.service.TelemetryRouterService;
-
-public class TelemetryRouterTask extends BaseSamzaTask {
-
-	static Logger LOGGER = new Logger(TelemetryRouterTask.class);
-	private TelemetryRouterConfig config;
-	private JobMetrics metrics;
-	private TelemetryRouterService service;
-
-	public TelemetryRouterTask(Config config, TaskContext context) {
-		init(config, context);
-	}
-
-	public TelemetryRouterTask() {
-
-	}
-
-	@Override
-	public void init(Config config, TaskContext context) {
-		
-		this.config = new TelemetryRouterConfig(config);
-		metrics = new JobMetrics(context, this.config.jobName());
-		service = new TelemetryRouterService(this.config);
-		this.initTask(config, metrics);
-	}
-
-	@Override
-	public void process(IncomingMessageEnvelope envelope, MessageCollector collector, TaskCoordinator taskCoordinator)
-			throws Exception {
-
-		TelemetryRouterSink sink = new TelemetryRouterSink(collector, metrics, config);
-		TelemetryRouterSource source = new TelemetryRouterSource(envelope);
-		service.process(source, sink);
-	}
-
-}
diff --git a/data-pipeline/telemetry-router/src/main/resources/log4j.xml b/data-pipeline/telemetry-router/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/telemetry-router/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index ce9a78a2cd..0000000000
--- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,340 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-
-import java.util.Map;
-
-public class EventFixture {
-
-	public static final String LOG_EVENT = "{\n" +
-            "  \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"data\": \"\",\n" +
-            "      \"err\": \"10\",\n" +
-            "      \"eventId\": \"GE_SIGNUP\",\n" +
-            "      \"id\": \"2131165210\",\n" +
-            "      \"type\": \"GENIE\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"2.2.15\"\n" +
-            "  },\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"2.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-	public static final String ERROR_EVENT = "{\n" +
-            "  \"did\": \"00b09a9e-6af9-4bb7-b102-57380b43ddc8\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"eid\": \"ERROR\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"2.2.15\"\n" +
-            "  },\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"2.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    }\n" +
-            "  ]\n" +
-            "}";
-	public static final String UNPARSABLE_START_EVENT = "{\n" +
-            "  \"did\": \"c270f15d-5230-4954-92aa-d239e4281cc4\",\n" +
-            "  \"mid\": \"43288930-e54a-230b-b56e-876gnm8712ok\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"mode\": \"WIFI\",\n" +
-            "      \"ver\": \"12\",\n" +
-            "      \"size\": 12.67,\n" +
-            "      \"err\": \"\",\n" +
-            "      \"referrer\": [\n" +
-            "        {\n" +
-            "          \"action\": \"INSTALL\",\n" +
-            "          \"utmsource\": \"Ekstep\",\n" +
-            "          \"utmmedium\": \"Portal\",\n" +
-            "          \"utmterm\": \"December 2016\",\n" +
-            "          \"utmcontent\": \"Ramayana\",\n" +
-            "          \"utmcampaign\": \"Epics of India\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"START\",\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"genie.android\",\n" +
-            "    \"ver\": \"1.0\"\n" +
-            "  },\n" +
-            "  \"sid\": \"\",\n" +
-            "  \"ets\": 1454064092546,\n" +
-            "  \"uid\": \"\",\n" +
-            "  \"ver\": \"2.0\",\n" +
-            "  \"cdata\": [\n" +
-            "    {\n" +
-            "      \"id\": \"correlationid\",\n" +
-            "      \"type\": \"correlationtype\"\n" +
-            "    ";
-	public static final String START_EVENT = "{\n" +
-            "  \"cdata\": [],\n" +
-            "  \"channel\": \"in.ekstep\",\n" +
-            "  \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" +
-            "  \"edata\": {\n" +
-            "    \"eks\": {\n" +
-            "      \"extype\": \"\",\n" +
-            "      \"id\": \"\",\n" +
-            "      \"pos\": [],\n" +
-            "      \"stageid\": \"Genie-TelemetrySync\",\n" +
-            "      \"subtype\": \"ManualSync-Success\",\n" +
-            "      \"tid\": \"\",\n" +
-            "      \"type\": \"OTHER\",\n" +
-            "      \"uri\": \"\",\n" +
-            "      \"values\": [\n" +
-            "        {\n" +
-            "          \"SizeOfFileInKB\": \"0.81\"\n" +
-            "        }\n" +
-            "      ]\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"eid\": \"GE_INTERACT\",\n" +
-            "  \"etags\": {\n" +
-            "    \"app\": []\n" +
-            "  },\n" +
-            "  \"ets\": 1506328824238,\n" +
-            "  \"gdata\": {\n" +
-            "    \"id\": \"org.ekstep.genieservices.qa\",\n" +
-            "    \"ver\": \"6.5.localdev-debug\"\n" +
-            "  },\n" +
-            "  \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" +
-            "  \"pdata\": {\n" +
-            "    \"id\": \"genie\",\n" +
-            "    \"ver\": \"6.5.localdev-debug\"\n" +
-            "  },\n" +
-            "  \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" +
-            "  \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" +
-            "  \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-            "  \"ver\": \"2.2\",\n" +
-            "  \"@version\": \"1\",\n" +
-            "  \"@timestamp\": \"2017-09-25T08:35:44.037Z\",\n" +
-            "  \"metadata\": {\n" +
-            "    \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" +
-            "  },\n" +
-            "  \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" +
-            "  \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-            "  \"type\": \"events\",\n" +
-            "  \"flags\": {\n" +
-            "    \"dd_processed\": true\n" +
-            "  }\n" +
-            "}";
-
-	public static final String AUDIT_EVENT ="{\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"Consumer\",\n" +
-			"    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-			"  },\n" +
-			"  \"eid\": \"AUDIT\",\n" +
-			"  \"edata\": {\n" +
-			"    \"state\": \"Create\",\n" +
-			"    \"props\": [\n" +
-			"      \"firstName\",\n" +
-			"      \"email\",\n" +
-			"      \"emailVerified\",\n" +
-			"      \"id\",\n" +
-			"      \"userId\",\n" +
-			"      \"createdBy\",\n" +
-			"      \"rootOrgId\",\n" +
-			"      \"channel\",\n" +
-			"      \"userType\",\n" +
-			"      \"roles\",\n" +
-			"      \"phoneVerified\",\n" +
-			"      \"isDeleted\",\n" +
-			"      \"createdDate\",\n" +
-			"      \"status\",\n" +
-			"      \"userName\",\n" +
-			"      \"loginId\",\n" +
-			"      \"externalIds\"\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"ets\": 1561739260405,\n" +
-			"  \"context\": {\n" +
-			"    \"channel\": \"0126684405014528002\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"pid\": \"learner-service\",\n" +
-			"      \"ver\": \"2.0.0\",\n" +
-			"      \"id\": \"prod.diksha.learning.service\"\n" +
-			"    },\n" +
-			"    \"env\": \"User\",\n" +
-			"    \"cdata\": [\n" +
-			"      {\n" +
-			"        \"type\": \"User\",\n" +
-			"        \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"SignupType\",\n" +
-			"        \"id\": \"google\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"Source\",\n" +
-			"        \"id\": \"android\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"Request\",\n" +
-			"        \"id\": \"a5f58660-99c1-11e9-8ef2-cdfa267be9f0\"\n" +
-			"      }\n" +
-			"    ],\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"0126684405014528002\"\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"mid\": \"1561739260405.1ab659ef-25b5-4a26-bd94-ff2daef86b62\",\n" +
-			"  \"object\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" +
-			"  },\n" +
-			"  \"syncts\": 1561739269966,\n" +
-			"  \"@timestamp\": \"2019-06-28T16:27:49.966Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-	public static final String START_EVENT_WITHOUT_ETS = "{\n" +
-			"  \"cdata\": [],\n" +
-			"  \"channel\": \"in.ekstep\",\n" +
-			"  \"did\": \"0427fedf56eea1c8a127d876fd1907ffb245684f\",\n" +
-			"  \"edata\": {\n" +
-			"    \"eks\": {\n" +
-			"      \"extype\": \"\",\n" +
-			"      \"id\": \"\",\n" +
-			"      \"pos\": [],\n" +
-			"      \"stageid\": \"Genie-TelemetrySync\",\n" +
-			"      \"subtype\": \"ManualSync-Success\",\n" +
-			"      \"tid\": \"\",\n" +
-			"      \"type\": \"OTHER\",\n" +
-			"      \"uri\": \"\",\n" +
-			"      \"values\": [\n" +
-			"        {\n" +
-			"          \"SizeOfFileInKB\": \"0.81\"\n" +
-			"        }\n" +
-			"      ]\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"eid\": \"GE_INTERACT\",\n" +
-			"  \"etags\": {\n" +
-			"    \"app\": []\n" +
-			"  },\n" +
-			"  \"gdata\": {\n" +
-			"    \"id\": \"org.ekstep.genieservices.qa\",\n" +
-			"    \"ver\": \"6.5.localdev-debug\"\n" +
-			"  },\n" +
-			"  \"mid\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\",\n" +
-			"  \"pdata\": {\n" +
-			"    \"id\": \"genie\",\n" +
-			"    \"ver\": \"6.5.localdev-debug\"\n" +
-			"  },\n" +
-			"  \"sid\": \"e9662bb5-cf08-42d6-ad11-700b23566961\",\n" +
-			"  \"ts\": \"2017-09-25T08:40:24.238+0000\",\n" +
-			"  \"uid\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-			"  \"ver\": \"2.2\",\n" +
-			"  \"@version\": \"1\",\n" +
-			"  \"@timestamp\": \"08:35:44.037\",\n" +
-			"  \"metadata\": {\n" +
-			"    \"checksum\": \"375f4921-bf7b-45b0-8cf1-ba92d876a815\"\n" +
-			"  },\n" +
-			"  \"uuid\": \"7f2c9f88-cbf7-4527-9f8d-667d4dde0d1c1\",\n" +
-			"  \"key\": \"03762014-f67b-466b-bf20-467f46542563\",\n" +
-			"  \"type\": \"events\",\n" +
-			"  \"flags\": {\n" +
-			"    \"dd_processed\": true\n" +
-			"  }\n" +
-			"}";
-
-	public static final String INVALID_EVENT ="{\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"Consumer\",\n" +
-			"    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-			"  },\n" +
-			"  \"edata\": {\n" +
-			"    \"state\": \"\",\n" +
-			"    \"props\": [\n" +
-			"    ]\n" +
-			"  },\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"ets\": 1561739260405,\n" +
-			"  \"context\": {\n" +
-			"    \"channel\": \"0126684405014528002\",\n" +
-			"    \"pdata\": {\n" +
-			"      \"pid\": \"learner-service\",\n" +
-			"      \"ver\": \"2.0.0\",\n" +
-			"      \"id\": \"prod.diksha.learning.service\"\n" +
-			"    },\n" +
-			"    \"env\": \"User\",\n" +
-			"    \"cdata\": [\n" +
-			"      {\n" +
-			"        \"type\": \"User\",\n" +
-			"        \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"SignupType\",\n" +
-			"        \"id\": \"google\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"Source\",\n" +
-			"        \"id\": \"android\"\n" +
-			"      },\n" +
-			"      {\n" +
-			"        \"type\": \"Request\",\n" +
-			"        \"id\": \"a5f58660-99c1-11e9-8ef2-cdfa267be9f0\"\n" +
-			"      }\n" +
-			"    ],\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"0126684405014528002\"\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"mid\": \"1561739260405.1ab659ef-25b5-4a26-bd94-ff2daef86b62\",\n" +
-			"  \"object\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"cf2bdab7-0778-4b8e-bfa5-fa7df36a5d19\"\n" +
-			"  },\n" +
-			"  \"syncts\": 1561739269966,\n" +
-			"  \"@timestamp\": \"2019-06-28T16:27:49.966Z\",\n" +
-			"  \"flags\": {\n" +
-			"    \"tv_processed\": true\n" +
-			"  },\n" +
-			"  \"type\": \"events\"\n" +
-			"}";
-
-	public static final String ANY_STRING = "Hey Samza, Whats Up?";
-	public static final String EMPTY_JSON = "{}";
-	public static final String SHARE_EVENT = "{\"ver\":\"3.0\",\"eid\":\"SHARE\",\"mid\":\"02ba33e5-15fe-4ec5-b360-3d03429fae84\",\"ets\":1577278681178,\"actor\":{\"type\":\"User\",\"id\":\"7c3ea1bb-4da1-48d0-9cc0-c4f150554149\"},\"context\":{\"cdata\":[{\"id\":\"1bfd99b0-2716-11ea-b7cc-13dec7acd2be\",\"type\":\"API\"},{\"id\":\"SearchResult\",\"type\":\"Section\"}],\"channel\":\"505c7c48ac6dc1edc9b08f21db5a571d\",\"pdata\":{\"id\":\"prod.diksha.app\",\"pid\":\"sunbird.app\",\"ver\":\"2.3.162\"},\"env\":\"app\",\"sid\":\"82e41d87-e33f-4269-aeae-d56394985599\",\"did\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\"},\"edata\":{\"dir\":\"In\",\"type\":\"File\",\"items\":[{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_312785709424099328114191\",\"type\":\"CONTENT\",\"ver\":\"1\",\"params\":[{\"transfers\":0,\"size\":21084308}]},{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_31277435209002188818711\",\"type\":\"CONTENT\",\"ver\":\"18\",\"params\":[{\"transfers\":12,\"size\":\"123\"}]},{\"origin\":{\"id\":\"1b17c32bad61eb9e33df281eecc727590d739b2b\",\"type\":\"Device\"},\"id\":\"do_31278794857559654411554\",\"type\":\"TextBook\",\"ver\":\"1\"}]},\"object\":{\"id\":\"do_312528116260749312248818\",\"type\":\"TextBook\",\"version\":\"10\",\"rollup\":{}},\"syncts\":1577278682630,\"@timestamp\":\"2019-12-25T12:58:02.630Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\"}";
-	
-	public static Map<String, Object> getMap(String message) {
-		return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-	}
-	
-}
diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java
deleted file mode 100644
index ef488947cd..0000000000
--- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/system/EventTest.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.system;
-
-import java.util.Map;
-
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.junit.Assert;
-import org.junit.Test;
-
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-
-public class EventTest {
-	
-	@Test
-	public void shouldReturnMid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.ERROR_EVENT));
-		Assert.assertEquals("43288930-e54a-230b-b56e-876gnm8712ok", event.mid());
-	}
-
-	@Test
-	public void shouldReturnEid() {
-
-		Event event = new Event(EventFixture.getMap(EventFixture.LOG_EVENT));
-		Assert.assertEquals("LOG", event.eid());
-	}
-
-	@Test(expected = JsonSyntaxException.class)
-	public void shouldThrowException() {
-		new Gson().fromJson(EventFixture.UNPARSABLE_START_EVENT,
-				new TypeToken<Map<String, Object>>() {
-				}.getType());
-	}
-}
diff --git a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java b/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java
deleted file mode 100644
index d502416c7d..0000000000
--- a/data-pipeline/telemetry-router/src/test/java/org/ekstep/ep/samza/task/TelemetryRouterTaskTest.java
+++ /dev/null
@@ -1,260 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-import org.mockito.Mockito;
-
-import java.lang.reflect.Type;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.Map;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class TelemetryRouterTaskTest {
-
-	private static final String PRIMARY_TOPIC = "telemetry.sink";
-	private static final String FAILED_TOPIC = "telemetry.failed";
-	private static final String SECONDARY_TOPIC = "telemetry.log";
-	private static final String MALFORMED_TOPIC = "telemetry.malformed";
-	private static final String AUDIT_TOPIC = "telemetry.audit";
-	private static final String SHARE_EVENT_TOPIC = "telemetry.share";
-	private static final String LOG_TOPIC = "events.log";
-	private static final String ERROR_TOPIC = "events.error";
-
-	private MessageCollector collectorMock;
-	private TaskContext contextMock;
-	private MetricsRegistry metricsRegistry;
-	private Counter counter;
-	private TaskCoordinator coordinatorMock;
-	private IncomingMessageEnvelope envelopeMock;
-	private Config configMock;
-	private TelemetryRouterTask telemetryRouterTask;
-
-	@Before
-	public void setUp() {
-		collectorMock = mock(MessageCollector.class);
-		contextMock = Mockito.mock(TaskContext.class);
-		metricsRegistry = Mockito.mock(MetricsRegistry.class);
-		counter = Mockito.mock(Counter.class);
-		coordinatorMock = mock(TaskCoordinator.class);
-		envelopeMock = mock(IncomingMessageEnvelope.class);
-		configMock = Mockito.mock(Config.class);
-
-		stub(configMock.get("router.events.primary.route.topic", PRIMARY_TOPIC)).toReturn(PRIMARY_TOPIC);
-		stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-		stub(configMock.get("router.events.secondary.route.topic", SECONDARY_TOPIC)).toReturn(SECONDARY_TOPIC);
-		stub(configMock.get("router.events.audit.route.topic", AUDIT_TOPIC)).toReturn(AUDIT_TOPIC);
-		stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-		stub(configMock.get("router.events.share.route.topic", SHARE_EVENT_TOPIC)).toReturn(SHARE_EVENT_TOPIC);
-		stub(configMock.get("router.events.log.topic.name", LOG_TOPIC)).toReturn(LOG_TOPIC);
-		stub(configMock.get("router.events.error.topic.name", ERROR_TOPIC)).toReturn(ERROR_TOPIC);
-
-		stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-		stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-		stub(envelopeMock.getOffset()).toReturn("2");
-		stub(envelopeMock.getSystemStreamPartition())
-				.toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1)));
-
-//		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-	}
-
-	@Test
-	public void shouldSendLOGEventToLogEventsRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.LOG_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), LOG_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendERROREventToPrimaryRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), PRIMARY_TOPIC)));
-
-	}
-
-	@Test
-	public void shouldSendERROREventToErrorEventsRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG,ERROR");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.ERROR_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), ERROR_TOPIC)));
-
-	}
-
-	@Test
-	public void shouldSendSTARTEventToPrimaryRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), PRIMARY_TOPIC)));
-
-	}
-
-	@Test
-	public void shouldSendAUDITEventToAuditRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-
-		verify(collectorMock, times(2)).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			int invocation = 0;
-
-			@Override
-			public boolean matches(Object o) {
-				invocation = invocation + 1;
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-				if (invocation == 1) {
-					assertEquals("kafka", systemStream.getSystem());
-					assertEquals(PRIMARY_TOPIC, systemStream.getStream());
-				}
-				if (invocation == 2) {
-					assertEquals("kafka", systemStream.getSystem());
-					assertEquals(AUDIT_TOPIC, systemStream.getStream());
-				}
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendSHAREEventToShareEventRoute() throws Exception {
-
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.SHARE_EVENT);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock, times(1)).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-				assertEquals("kafka", systemStream.getSystem());
-				assertEquals(SHARE_EVENT_TOPIC, systemStream.getStream());
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldStampTSForPrimaryEvent() throws Exception {
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT);
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		Type mapType = new TypeToken<Map<String, Object>>() {
-		}.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				assertNotNull(outputEvent.get("ts"));
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventToFailedTopicIfEventIsNotParseable() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSABLE_START_EVENT);
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	@Test
-	public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception {
-
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING);
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-	}
-
-	@Test
-	public void shouldAddTodayDateIfETSIsNotPresent() throws Exception {
-		stub(configMock.get("router.events.secondary.route.events", "LOG,ERROR")).toReturn("LOG");
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.START_EVENT_WITHOUT_ETS);
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		telemetryRouterTask.process(envelopeMock, collectorMock, coordinatorMock);
-		String simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm").format(new Date());
-		Type mapType = new TypeToken<Map<String, Object>>() {
-		}.getType();
-		verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-				Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-				String updatedTS = outputEvent.get("ts").toString();
-				assertNotNull(updatedTS);
-				assertEquals(updatedTS.substring(0,updatedTS.indexOf(".")-3), simpleDateFormat);
-				return true;
-			}
-		}));
-	}
-
-	@Test
-	public void shouldSendEventsToErrorTopic() throws Exception {
-		stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_EVENT);
-		telemetryRouterTask = new TelemetryRouterTask(configMock, contextMock);
-		telemetryRouterTask.process(envelopeMock,collectorMock, coordinatorMock);
-		verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-	}
-
-
-	public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-		return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-			@Override
-			public boolean matches(Object o) {
-				OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-				SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-				assertEquals("kafka", systemStream.getSystem());
-				assertEquals(stream, systemStream.getStream());
-
-				return true;
-			}
-		};
-	}
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json
deleted file mode 100755
index 4c7261b025..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json b/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json
deleted file mode 100755
index 9ac4103bd6..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/common_empty_sid_uid.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "pdata": {
-      "$ref": "file:src/test/resources/2.2/pdata.json"
-    },
-    "gdata": {
-      "$ref": "file:src/test/resources/2.2/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "file:src/test/resources/2.2/etags.json"
-    },
-    "cdata": {
-      "$ref": "file:src/test/resources/2.2/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/2.2/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json b/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json
deleted file mode 100755
index e00c02e041..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/etags.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "type": "object",
-    "properties": {
-      "app": {
-        "id": "http://api.ekstep.org/telemetry/etags/app",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/app/items",
-            "type": "string"
-        }
-      },
-      "partner": {
-        "id": "http://api.ekstep.org/telemetry/etags/partner",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/partner/items",
-            "type": "string"
-        }
-      },
-      "dims": {
-        "id": "http://api.ekstep.org/telemetry/etags/dims",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/dims/items",
-            "type": "string"
-        }
-      }
-    },
-    "additionalProperties": false
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json
deleted file mode 100755
index 1b603a1b7b..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json b/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json
deleted file mode 100755
index 14350be425..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/ge_error.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "file:src/test/resources/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_ERROR"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "id",
-                                "err",
-                                "eventId",
-                                "data"
-                            ],
-                            "properties": {
-                                "err": {
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "SYSTEM",
-                                        "GENIE",
-                                        "GAME",
-                                        "GENIESERVICES"
-                                    ]
-                                },
-                                "id": {
-                                    "type": "string"
-                                },
-                                "eventId": {
-                                    "type": "string"
-                                },
-                                "data": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
diff --git a/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json b/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json
deleted file mode 100755
index cd509bc58b..0000000000
--- a/data-pipeline/telemetry-router/src/test/resources/2.2/pdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/pdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/pdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/data-pipeline/telemetry-validator/pom.xml b/data-pipeline/telemetry-validator/pom.xml
deleted file mode 100644
index e3212db210..0000000000
--- a/data-pipeline/telemetry-validator/pom.xml
+++ /dev/null
@@ -1,188 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-    <prerequisites>
-        <maven>3.0.1</maven>
-    </prerequisites>
-
-    <parent>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <artifactId>jobs</artifactId>
-        <version>0.0.1</version>
-    </parent>
-
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>telemetry-validator</artifactId>
-    <version>0.2.2</version>
-    <packaging>jar</packaging>
-    <name>TelemetryValidator</name>
-    <description>
-        Validating Event Against Schema
-    </description>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-         <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.schwering</groupId>
-            <artifactId>irclib</artifactId>
-            <version>1.10</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.cedarsoftware</groupId>
-            <artifactId>json-io</artifactId>
-            <version>4.0.0</version>
-        </dependency>
-        <!--
-        <dependency>
-            <groupId>com.google.maps</groupId>
-            <artifactId>google-maps-services</artifactId>
-            <version>0.1.7</version>
-        </dependency>
-        -->
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>com.github.java-json-tools</groupId>
-            <artifactId>json-schema-validator</artifactId>
-            <version>2.2.8</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>joda-time</groupId>
-                    <artifactId>joda-time</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-</project>
diff --git a/data-pipeline/telemetry-validator/src/main/assembly/src.xml b/data-pipeline/telemetry-validator/src/main/assembly/src.xml
deleted file mode 100644
index 0d5552e0a4..0000000000
--- a/data-pipeline/telemetry-validator/src/main/assembly/src.xml
+++ /dev/null
@@ -1,69 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/telemetry-validator.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:telemetry-validator</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties b/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties
deleted file mode 100644
index 910780c7c9..0000000000
--- a/data-pipeline/telemetry-validator/src/main/config/telemetry-validator.properties
+++ /dev/null
@@ -1,73 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.TelemetryValidator
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__telemetry_validator_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.TelemetryValidatorTask
-task.inputs=kafka.__env__.telemetry.raw
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-output.success.topic.name=__env__.telemetry.valid
-output.failed.topic.name=__env__.telemetry.failed
-output.malformed.topic.name=__env__.telemetry.malformed
-output.metrics.topic.name=__env__.pipeline_metrics
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,failed-message-count,error-message-count
-default.channel=__default_channel__
-
-telemetry.schema.path=__telemetry_schema_path__
-telemetry.schema.version=3.0
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index b9c85cf6ae..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,98 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.apache.commons.lang.StringUtils;
-import org.ekstep.ep.samza.events.domain.Events;
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.task.TelemetryValidatorConfig;
-import org.joda.time.format.DateTimeFormat;
-import org.joda.time.format.DateTimeFormatter;
-
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-
-public class Event extends Events {
-
-    private DateTimeFormatter df = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'").withZoneUTC();
-
-    public Event(Map<String, Object> map) {
-        super(map);
-    }
-
-    public String schemaName() {
-        String eid = eid();
-        if (eid != null) {
-            return MessageFormat.format("{0}.json", eid.toLowerCase());
-        } else {
-            return "envelope.json";
-        }
-    }
-
-    public void markSuccess() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tv_processed", true);
-        telemetry.add("type", "events");
-    }
-
-    public void markFailure(String error, TelemetryValidatorConfig config) {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tv_processed", false);
-
-        telemetry.addFieldIfAbsent("metadata", new HashMap<String, Object>());
-        if (null != error) {
-            telemetry.add("metadata.tv_error", error);
-            telemetry.add("metadata.src", config.jobName());
-        }
-
-    }
-
-    public void markSkipped() {
-        telemetry.addFieldIfAbsent("flags", new HashMap<String, Boolean>());
-        telemetry.add("flags.tv_skipped", true);
-    }
-
-    public void updateActorId(String actorId) {
-        telemetry.add("actor.id", actorId);
-    }
-
-    public void correctDialCodeKey() {
-        NullableValue<Object> dialcodes = telemetry.read("edata.filters.dialCodes");
-        if (dialcodes != null && dialcodes.value() != null) {
-            telemetry.add("edata.filters.dialcodes", dialcodes.value());
-            telemetry.add("edata.filters.dialCodes", null);
-        }
-    }
-
-    public void correctDialCodeValue() {
-        String dialcode = telemetry.<String>read("object.id").value();
-        telemetry.add("object.id", dialcode.toUpperCase());
-    }
-
-    public void updateDefaults(TelemetryValidatorConfig config) {
-        String channelString = telemetry.<String>read("context.channel").value();
-        String channel = StringUtils.deleteWhitespace(channelString);
-        if (channel == null || channel.isEmpty()) {
-            telemetry.addFieldIfAbsent("context", new HashMap<String, Object>());
-            telemetry.add("context.channel", config.defaultChannel());
-        }
-        String atTimestamp = telemetry.getAtTimestamp();
-        String strSyncts = telemetry.getSyncts();
-        if (null == atTimestamp && null == strSyncts) {
-            long syncts = System.currentTimeMillis();
-            telemetry.addFieldIfAbsent("syncts", syncts);
-            telemetry.addFieldIfAbsent("@timestamp", df.print(syncts));
-        } else {
-            if (atTimestamp != null) {
-                telemetry.addFieldIfAbsent("syncts", df.parseMillis(atTimestamp));
-            } else if (strSyncts != null) {
-                telemetry.addFieldIfAbsent("@timestamp", strSyncts);
-            }
-        }
-
-    }
-
-    public String actorId() {
-        NullableValue<String> checksum = telemetry.read("actor.id");
-        return checksum.value();
-    }
-}
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java
deleted file mode 100644
index f3969b0b92..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/service/TelemetryValidatorService.java
+++ /dev/null
@@ -1,95 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.github.fge.jsonschema.core.report.ProcessingReport;
-import com.google.gson.JsonSyntaxException;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.TelemetryValidatorConfig;
-import org.ekstep.ep.samza.task.TelemetryValidatorSink;
-import org.ekstep.ep.samza.task.TelemetryValidatorSource;
-import org.ekstep.ep.samza.util.TelemetrySchemaValidator;
-
-import static java.text.MessageFormat.format;
-
-public class TelemetryValidatorService {
-    private static Logger LOGGER = new Logger(TelemetryValidatorService.class);
-    private final TelemetryValidatorConfig config;
-    private TelemetrySchemaValidator telemetrySchemaValidator;
-    private String eidNullErrorMsg = "VALIDATION FAILED as eid is missing";
-    private String schemaNotFoundErrorMsg = "SCHEMA NOT FOUND";
-
-    public TelemetryValidatorService(TelemetryValidatorConfig config, TelemetrySchemaValidator telemetrySchemaValidator) {
-        this.config = config;
-        this.telemetrySchemaValidator = telemetrySchemaValidator;
-    }
-
-    public void process(TelemetryValidatorSource source, TelemetryValidatorSink sink) {
-        Event event = null;
-        try {
-            event = source.getEvent();
-            if(event.eid() == null) {
-                LOGGER.error(null, eidNullErrorMsg);
-                sink.toFailedTopic(event, eidNullErrorMsg);
-                return;
-            }
-
-            if (!telemetrySchemaValidator.schemaFileExists(event)) {
-                LOGGER.debug(null, schemaNotFoundErrorMsg);
-                sink.toFailedTopic(event, schemaNotFoundErrorMsg);
-                return;
-            }
-
-            ProcessingReport report = telemetrySchemaValidator.validate(event);
-
-            if (report.isSuccess()) {
-                LOGGER.debug("VALIDATION SUCCESS", event.mid());
-                event.markSuccess();
-                event.updateDefaults(config);
-                sink.toSuccessTopic(dataCorrection(event));
-            } else {
-                LOGGER.error(null, "VALIDATION FAILED: " + report.toString());
-                String fieldName = getInvalidFieldName(report.toString());
-                sink.toFailedTopic(event, String.format("validation failed. fieldname: %s", fieldName));
-            }
-        } catch (JsonSyntaxException e) {
-            LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-            sink.toMalformedEventsTopic(source.getMessage());
-        } catch (Exception e) {
-            LOGGER.error(null, format(
-                    "EXCEPTION. PASSING EVENT THROUGH AND ADDING IT TO EXCEPTION TOPIC. EVENT: {0}, EXCEPTION:",
-                    event), e);
-            sink.toErrorTopic(event, e.getMessage());
-        }
-    }
-
-    public String getInvalidFieldName(String errorInfo) {
-        String[] message = errorInfo.split("reports:");
-        String noFieldNameMsg = "unable to get the field name";
-        if (message.length > 1) {
-            String[] fields = message[1].split(",");
-            if (fields.length > 2) {
-                String[] pointer = fields[3].split("\"pointer\":");
-                return pointer[1].substring(0, pointer[1].length() - 1);
-            }
-        }
-        return noFieldNameMsg;
-    }
-
-    public Event dataCorrection(Event event) {
-        // Remove prefix from federated userIds
-        String eventActorId = event.actorId();
-        if (eventActorId != null && !eventActorId.isEmpty() && eventActorId.startsWith("f:")) {
-            event.updateActorId(eventActorId.substring(eventActorId.lastIndexOf(":") + 1));
-        }
-
-        if (event.eid() != null && event.eid().equalsIgnoreCase("SEARCH")) {
-            event.correctDialCodeKey();
-        }
-
-        if (event.objectFieldsPresent() && (event.objectType().equalsIgnoreCase("DialCode") || event.objectType().equalsIgnoreCase("qr"))) {
-            event.correctDialCodeValue();
-        }
-        return event;
-    }
-
-}
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java
deleted file mode 100644
index 8ac679cd08..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorConfig.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import org.apache.samza.config.Config;
-
-public class TelemetryValidatorConfig {
-
-    private final String JOB_NAME = "TelemetryValidator";
-    private String schemaPath;
-    private String successTopic;
-    private String failedTopic;
-    private String malformedTopic;
-    private String defaultChannel;
-    private String schema_version;
-
-    public TelemetryValidatorConfig(Config config) {
-        successTopic = config.get("output.success.topic.name", "telemetry.valid");
-        failedTopic = config.get("output.failed.topic.name", "telemetry.failed");
-        malformedTopic = config.get("output.malformed.topic.name", "telemetry.malformed");
-        defaultChannel = config.get("default.channel", "org.sunbird");
-        schemaPath = config.get("telemetry.schema.path", "/etc/samza-jobs/schemas");
-        schema_version = config.get("telemetry.schema.version", "3.0");
-
-    }
-
-    public String successTopic() {
-        return successTopic;
-    }
-
-    public String failedTopic() {
-        return failedTopic;
-    }
-
-    public String malformedTopic() {
-        return malformedTopic;
-    }
-
-    public String defaultChannel() {
-        return defaultChannel;
-    }
-
-    public String schemaPath() {
-        return schemaPath;
-    }
-
-    public String schemaVersion() {
-        return schema_version;
-    }
-
-    public String jobName() {
-        return JOB_NAME;
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java
deleted file mode 100644
index 870c24b26e..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSink.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.domain.Event;
-
-public class TelemetryValidatorSink extends BaseSink {
-
-    private TelemetryValidatorConfig config;
-
-    public TelemetryValidatorSink(MessageCollector collector, JobMetrics metrics,
-                                  TelemetryValidatorConfig config) {
-        
-    	super(collector, metrics);
-        this.config = config;
-    }
-
-    public void toSuccessTopic(Event event) {
-        toTopic(config.successTopic(), event.mid(), event.getJson());
-        metrics.incSuccessCounter();
-    }
-
-    public void toFailedTopic(Event event, String failedMessage) {
-    	event.markFailure(failedMessage, config);
-    	toTopic(config.failedTopic(), event.mid(), event.getJson());
-        metrics.incFailedCounter();
-    }
-
-    public void toErrorTopic(Event event, String errorMessage) {
-    	event.markFailure(errorMessage, config);
-    	toTopic(config.failedTopic(), event.mid(), event.getJson());
-        metrics.incErrorCounter();
-    }
-
-    public void toMalformedEventsTopic(String message) {
-        toTopic(config.malformedTopic(), null, message);
-        metrics.incFailedCounter();
-    }
-
-}
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java
deleted file mode 100644
index 277b73bf4d..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorSource.java
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Map;
-
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-
-import com.google.gson.Gson;
-
-public class TelemetryValidatorSource {
-    static Logger LOGGER = new Logger(TelemetryValidatorSource.class);
-
-    private IncomingMessageEnvelope envelope;
-
-    public TelemetryValidatorSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        return new Event(getMap());
-    }
-
-    @SuppressWarnings("unchecked")
-    private Map<String, Object> getMap() {
-        String message = (String) envelope.getMessage();
-        return (Map<String, Object>) new Gson().fromJson(message, Map.class);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-
-
-}
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java
deleted file mode 100644
index de41d8d09a..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/task/TelemetryValidatorTask.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.TelemetryValidatorService;
-import org.ekstep.ep.samza.util.TelemetrySchemaValidator;
-
-public class TelemetryValidatorTask extends BaseSamzaTask {
-
-    private TelemetryValidatorConfig config;
-    private JobMetrics metrics;
-    private TelemetryValidatorService service;
-
-    public TelemetryValidatorTask(Config config, TaskContext context, TelemetrySchemaValidator telemetrySchemaValidator) throws Exception {
-        init(config, context, telemetrySchemaValidator);
-    }
-
-    public TelemetryValidatorTask() {
-
-    }
-
-
-    @Override
-    public void init(Config config, TaskContext context) throws Exception {
-        init(config, context, null);
-    }
-
-
-    public void init(Config config, TaskContext context, TelemetrySchemaValidator schemaValidator) throws Exception {
-        this.config = new TelemetryValidatorConfig(config);
-        metrics = new JobMetrics(context, this.config.jobName());
-        TelemetrySchemaValidator telemetrySchemaValidator = schemaValidator == null ? new TelemetrySchemaValidator(this.config) : schemaValidator;
-        service = new TelemetryValidatorService(this.config, telemetrySchemaValidator);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-        TelemetryValidatorSource source = new TelemetryValidatorSource(envelope);
-        TelemetryValidatorSink sink = new TelemetryValidatorSink(collector, metrics, config);
-
-        service.process(source, sink);
-    }
-   
-}
diff --git a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java b/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java
deleted file mode 100644
index 3ef33ca3d5..0000000000
--- a/data-pipeline/telemetry-validator/src/main/java/org/ekstep/ep/samza/util/TelemetrySchemaValidator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.ekstep.ep.samza.util;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.github.fge.jackson.JsonLoader;
-import com.github.fge.jsonschema.core.exceptions.ProcessingException;
-import com.github.fge.jsonschema.core.report.ProcessingReport;
-import com.github.fge.jsonschema.main.JsonSchema;
-import com.github.fge.jsonschema.main.JsonSchemaFactory;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.TelemetryValidatorConfig;
-
-import java.io.File;
-import java.io.IOException;
-import java.text.MessageFormat;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TelemetrySchemaValidator {
-
-    private Map<String, JsonSchema> schemaJsonMap = new HashMap<>();
-
-    public TelemetrySchemaValidator(TelemetryValidatorConfig config) throws IOException, ProcessingException {
-
-        String schemaVersion = config.schemaVersion();
-        File schemaDirectory = new File(MessageFormat.format("{0}/{1}/", config.schemaPath(), schemaVersion));
-        File[] schemaFiles = schemaDirectory.listFiles();
-        for (File schemafile : schemaFiles) {
-            String schemaKey = String.format("%s", schemafile.getName());
-            schemaJsonMap.put(schemaKey, JsonSchemaFactory
-                    .byDefault().getJsonSchema(JsonLoader.fromFile(schemafile)));
-
-        }
-
-    }
-
-    public boolean schemaFileExists(Event event) {
-        return schemaJsonMap.containsKey(String.format("%s", event.schemaName()));
-    }
-
-    public ProcessingReport validate(Event event) throws IOException, ProcessingException {
-        JsonNode eventJson = JsonLoader.fromString(event.getJson());
-        String schemaKey = String.format("%s", event.schemaName());
-        ProcessingReport report = schemaJsonMap.get(schemaKey).validate(eventJson);
-        return report;
-    }
-
-}
diff --git a/data-pipeline/telemetry-validator/src/main/resources/log4j.xml b/data-pipeline/telemetry-validator/src/main/resources/log4j.xml
deleted file mode 100644
index e416dd51b3..0000000000
--- a/data-pipeline/telemetry-validator/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-  <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-  <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-     <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-      <param name="append" value="true" />
-      <param name="maxFileSize" value="100MB" />
-      <param name="maxBackupIndex" value="5" />
-     <layout class="org.apache.log4j.PatternLayout">
-      <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-     </layout>
-  </appender>
-  <root>
-    <priority value="info" />
-    <appender-ref ref="RollingAppender"/>
-    <appender-ref ref="jmx" />
-  </root>
-</log4j:configuration>
diff --git a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java b/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
deleted file mode 100644
index 98544883d6..0000000000
--- a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/fixtures/EventFixture.java
+++ /dev/null
@@ -1,597 +0,0 @@
-package org.ekstep.ep.samza.fixtures;
-
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-
-import java.util.Map;
-
-public class EventFixture {
-
-	public static final String VALID_IMPRESSION_EVENT = "{\"actor\":{\"type\":\"User\",\"id\":\"f:5a8a3f2b-3409-42e0-9001-f913bc0fde31:874ed8a5-782e-4f6c-8f36-e0288455901e\"},\"eid\":\"IMPRESSION\",\"edata\":{\"type\":\"detail\",\"pageid\":\"content-detail\",\"uri\":\"content-detail\"},\"ver\":\"3.0\",\"ets\":1.592591395144E12,\"context\":{\"pdata\":{\"pid\":\"sunbird.app\",\"ver\":\"2.10.307\",\"id\":\"prod.diksha.app\"},\"channel\":\"0126684405014528002\",\"env\":\"home\",\"did\":\"77f505f41501d86a0ccaec9cf3b9ec99bec5af06\",\"cdata\":[],\"sid\":\"57b8bc17-1705-4cdd-885b-99db23f4b36b\",\"rollup\":{\"l1\":\"0126684405014528002\"}},\"mid\":\"f5114fee-ab6b-42da-afbe-13bd2bdb0476\",\"object\":{\"id\":\"do_3130160314974781441206\",\"type\":\"ExplanationResource\",\"version\":\"1\",\"rollup\":{\"l1\":\"do_3130160314974781441206\"}},\"syncts\":1592591400233,\"@timestamp\":\"2020-06-19T18:30:00.233Z\"}";
-
-	public static final String UNPARSEABLE_EVENT = "{\n" +
-			"  \"actor\": {\n" +
-			"    \"type\": \"User\",\n" +
-			"    \"id\": \"5a75030f-79be-4a08-a6b2-eb27769a23f5\"\n" +
-			"  },\n" +
-			"  \"eid\": \"IMPRESSION\",\n" +
-			"  \"edata\": {\n" +
-			"    \"type\": \"detail\",\n" +
-			"    \"pageid\": \"content-detail\",\n" +
-			"    \"uri\": \"content-detail\"\n" +
-			"  },\n" +
-			"  \"ver\": \"3.0\",\n" +
-			"  \"ets\": 1.592591395144E12,\n" +
-			"  \"context\": {\n" +
-			"    \"pdata\": {\n" +
-			"      \"pid\": \"sunbird.app\",\n" +
-			"      \"ver\": \"2.10.307\",\n" +
-			"      \"id\": \"prod.diksha.app\"\n" +
-			"    },\n" +
-			"    \"channel\": \"0126684405014528002\",\n" +
-			"    \"env\": \"home\",\n" +
-			"    \"did\": \"77f505f41501d86a0ccaec9cf3b9ec99bec5af06\",\n" +
-			"    \"cdata\": [\n" +
-			"      \n" +
-			"    ],\n" +
-			"    \"sid\": \"57b8bc17-1705-4cdd-885b-99db23f4b36b\",\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"0126684405014528002\"\n" +
-			"    }\n" +
-			"  \"mid\": \"f5114fee-ab6b-42da-afbe-13bd2bdb0476\",\n" +
-			"  \"object\": {\n" +
-			"    \"id\": \"do_3130160314974781441206\",\n" +
-			"    \"type\": \"ExplanationResource\",\n" +
-			"    \"version\": \"1\",\n" +
-			"    \"rollup\": {\n" +
-			"      \"l1\": \"do_3130160314974781441206\"\n" +
-			"    }\n" +
-			"  },\n" +
-			"  \"syncts\": 1592591400233,\n" +
-			"  \"@timestamp\": \"2020-06-19T18:30:00.233Z\"\n" +
-			"}";
-
-	public static final String SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY = "{\n" +
-            "  \"eid\": \"SEARCH\",\n" +
-            "  \"ets\": 1.554241415598E12,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"LP.1554241415598.bff9398b-a827-41ec-b1c9-e569546174b1\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"org.ekstep.learning.platform\",\n" +
-            "    \"type\": \"System\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"in.ekstep\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.ntp.learning.platform\",\n" +
-            "      \"pid\": \"search-service\",\n" +
-            "      \"ver\": \"1.0\"\n" +
-            "    },\n" +
-            "    \"env\": \"search\"\n" +
-            "  },\n" +
-            "  \"edata\": {\n" +
-            "    \"size\": 499413.0,\n" +
-            "    \"query\": \"\",\n" +
-            "    \"filters\": {\n" +
-            "      \"dialCodes\": [\n" +
-            "        \"ATAZRI\",\n" +
-            "        \"CGSX7D\"\n" +
-            "      ]\n" +
-            "    },\n" +
-            "    \"sort\": {\n" +
-            "      \n" +
-            "    },\n" +
-            "    \"type\": \"all\",\n" +
-            "    \"topn\": [\n" +
-            "      {\n" +
-            "        \"identifier\": \"do_31268582767737241615189\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"identifier\": \"do_31269107959395942417491\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"identifier\": \"do_31269108472948326417493\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"identifier\": \"do_31269113788995174417318\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"identifier\": \"do_31270597860728832015700\"\n" +
-            "      }\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\",\n" +
-            "  \"syncts\": 1.55424141664E12,\n" +
-            "  \"@timestamp\": \"2019-04-02T21:43:36.640Z\"\n" +
-            "}";
-
-	public static final String EVENT_WITH_EID_MISSING = "{\n" +
-            "  \"ver\":\"3.0\",\n" +
-            "  \"syncts\":0.0,\n" +
-            "  \"ets\":0.0,\n" +
-            "  \"tags\":[\n" +
-            "\n" +
-            "  ],\n" +
-            "  \"context\":{\n" +
-            "    \"channel\":\"01250894314817126443\"\n" +
-            "  },\n" +
-            "  \"ts\":\"1970-01-01T00:00:00.000+0000\",\n" +
-            "  \"metadata\":{\n" +
-            "    \"checksum\":\"10b890f9783ddcae406e7d6514d073eed4327941\",\n" +
-            "    \"odn_status\":\"failed\",\n" +
-            "    \"odn_error\":null\n" +
-            "  }\n" +
-            "}";
-
-    public static final String INVALID_EVENT = "{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"393407b1-66b1-4c86-9080-b2bce9842886\"\n" +
-            "  },\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"edata\": {\n" +
-            "    \"id\": \"ContentDetail\",\n" +
-            "    \"pageid\": \"ContentDetail\",\n" +
-            "    \"type\": \"TOUCH\",\n" +
-            "    \"subtype\": \"ContentDownload-Initiate\"\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": \"1541574545180\",\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"ver\": \"2.1.8\",\n" +
-            "      \"pid\": \"sunbird.app\",\n" +
-            "      \"id\": \"prod.diksha.app\"\n" +
-            "    },\n" +
-            "    \"channel\": \"0123221617357783046602\",\n" +
-            "    \"env\": \"sdk\",\n" +
-            "    \"did\": \"68dfc64a7751ad47617ac1a4e0531fb761ebea6f\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"qr\",\n" +
-            "        \"id\": \"K4KCXE\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"API\",\n" +
-            "        \"id\": \"f3ac6610-d218-11e8-b2bb-1598ac1fcb99\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"sid\": \"70ea93d0-e521-4030-934f-276e7194c225\"\n" +
-            "  },\n" +
-            "  \"mid\": \"e6a3bcd3-eb78-457b-8fc0-4acc94642ebf\",\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"do_31249561779090227216256\",\n" +
-            "    \"type\": \"Content\",\n" +
-            "    \"version\": \"\"\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"syncts\": 1539846605341,\n" +
-            "  \"@timestamp\": \"2018-10-18T07:10:05.341Z\",\n" +
-            "  \"derivedlocationdata\": {\n" +
-            "    \"district\": \"Bengaluru\",\n" +
-            "    \"state\": 1234,\n" +
-            "    \"from\": \"user-profile\"\n" +
-            "  },\n" +
-            "  \"devicedata\": {\n" +
-            "    \"statecustomcode\": \"KA-Custom\",\n" +
-            "    \"country\": \"India\",\n" +
-            "    \"iso3166statecode\": \"IN-KA\",\n" +
-            "    \"city\": \"Bangalore\",\n" +
-            "    \"countrycode\": \"IN\",\n" +
-            "    \"statecode\": \"KA\",\n" +
-            "    \"devicespec\": {\n" +
-            "      \"os\": \"Android 6.0\",\n" +
-            "      \"make\": \"Motorola XT1706\"\n" +
-            "    },\n" +
-            "    \"districtcustom\": \"Banglore-Custom\",\n" +
-            "    \"firstaccess\": 1559484698000,\n" +
-            "    \"uaspec\": {\n" +
-            "      \"agent\": \"Mozilla\",\n" +
-            "      \"ver\": \"5.0\",\n" +
-            "      \"system\": \"iPad\",\n" +
-            "      \"raw\": \"Mozilla/5.0 (X11 Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko)\",\n" +
-            "      \"platform\": \"AppleWebKit/531.21.10\"\n" +
-            "    },\n" +
-            "    \"state\": \"Karnataka\",\n" +
-            "    \"statecustomname\": \"Karnatak-Custom\",\n" +
-            "    \"userdeclared\": {\n" +
-            "      \"district\": \"Bangalore\",\n" +
-            "      \"state\": \"Karnataka\"\n" +
-            "    }\n" +
-            "  }\n" +
-            "}";
-
-    public static final String  VALID_LOG_EVENT = "{\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"ets\": 1570817279146,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"1\",\n" +
-            "    \"type\": \"service\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.content-service\",\n" +
-            "      \"ver\": \"1.0\",\n" +
-            "      \"pid\": \"sunbird-content-service\"\n" +
-            "    },\n" +
-            "    \"env\": \"content\",\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"do_312776645813747712111638\",\n" +
-            "    \"type\": \"content\",\n" +
-            "    \"ver\": \"\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"api_access\",\n" +
-            "    \"level\": \"INFO\",\n" +
-            "    \"message\": \"successful\",\n" +
-            "    \"params\": [\n" +
-            "      {\n" +
-            "        \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"title\": \"Content read api\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"category\": \"contentread\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"url\": \"content/read\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"method\": \"GET\"\n" +
-            "      }\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"syncts\": 1570817431399,\n" +
-            "  \"@timestamp\": \"2019-10-11T18:10:31.399Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String VALID_LOG_WITH_MISSING_CHANNEL="{\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"ets\": 1570817279146,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"1\",\n" +
-            "    \"type\": \"service\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.content-service\",\n" +
-            "      \"ver\": \"1.0\",\n" +
-            "      \"pid\": \"sunbird-content-service\"\n" +
-            "    },\n" +
-            "    \"env\": \"content\",\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"do_312776645813747712111638\",\n" +
-            "    \"type\": \"content\",\n" +
-            "    \"ver\": \"\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"api_access\",\n" +
-            "    \"level\": \"INFO\",\n" +
-            "    \"message\": \"successful\",\n" +
-            "    \"params\": [\n" +
-            "      {\n" +
-            "        \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"title\": \"Content read api\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"category\": \"contentread\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"url\": \"content/read\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"method\": \"GET\"\n" +
-            "      }\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"syncts\": 1570817431399,\n" +
-            "  \"@timestamp\": \"2019-10-11T18:10:31.399Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String VALID_LOG_WITH_MISSING_SYNCTS="{\n" +
-            "  \"eid\": \"LOG\",\n" +
-            "  \"ets\": 1570817279146,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"LOG:2a210813fac274656f194f9807ad8abf\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"1\",\n" +
-            "    \"type\": \"service\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"prod.diksha.content-service\",\n" +
-            "      \"ver\": \"1.0\",\n" +
-            "      \"pid\": \"sunbird-content-service\"\n" +
-            "    },\n" +
-            "    \"env\": \"content\",\n" +
-            "    \"sid\": \"\",\n" +
-            "    \"did\": \"1e465bed2138b8f2764d946492293e2b2f628789\",\n" +
-            "    \"cdata\": [],\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"do_312776645813747712111638\",\n" +
-            "    \"type\": \"content\",\n" +
-            "    \"ver\": \"\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"tags\": [],\n" +
-            "  \"edata\": {\n" +
-            "    \"type\": \"api_access\",\n" +
-            "    \"level\": \"INFO\",\n" +
-            "    \"message\": \"successful\",\n" +
-            "    \"params\": [\n" +
-            "      {\n" +
-            "        \"rid\": \"0ece8640-ec52-11e9-a3d6-697ee5684e40\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"title\": \"Content read api\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"category\": \"contentread\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"url\": \"content/read\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"method\": \"GET\"\n" +
-            "      }\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_LOWERCASE = "{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"anonymous\"\n" +
-            "    },\n" +
-            "    \"eid\": \"IMPRESSION\",\n" +
-            "    \"edata\": {\n" +
-            "        \"visits\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"type\": \"view\",\n" +
-            "        \"pageid\": \"dialcode\",\n" +
-            "        \"subtype\": \"pageexit\",\n" +
-            "        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": 1570817279146,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"anonymous\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.7.1\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"dev.sunbird.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"env\": \"public\",\n" +
-            "        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-            "        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"977d3i\",\n" +
-            "        \"type\": \"dialcode\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1550501698819,\n" +
-            "    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-            "    \"ldata\": {\n" +
-            "        \"country_code\": \"\",\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"state_code\": \"\"\n" +
-            "    }  \n" +
-            "}";
-
-    public static final String IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_UPPERCASE = "{\n" +
-            "    \"actor\": {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"anonymous\"\n" +
-            "    },\n" +
-            "    \"eid\": \"IMPRESSION\",\n" +
-            "    \"edata\": {\n" +
-            "        \"visits\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"type\": \"view\",\n" +
-            "        \"pageid\": \"dialcode\",\n" +
-            "        \"subtype\": \"pageexit\",\n" +
-            "        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-            "    },\n" +
-            "    \"ver\": \"3.0\",\n" +
-            "    \"ets\": 1570817279146,\n" +
-            "    \"context\": {\n" +
-            "        \"uid\": \"anonymous\",\n" +
-            "        \"pdata\": {\n" +
-            "            \"ver\": \"1.7.1\",\n" +
-            "            \"pid\": \"sunbird-portal\",\n" +
-            "            \"id\": \"dev.sunbird.portal\"\n" +
-            "        },\n" +
-            "        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "        \"env\": \"public\",\n" +
-            "        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-            "        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-            "        \"cdata\": [\n" +
-            "            \n" +
-            "        ],\n" +
-            "        \"rollup\": {\n" +
-            "            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-            "    \"object\": {\n" +
-            "        \"ver\": \"1.0\",\n" +
-            "        \"id\": \"977D3I\",\n" +
-            "        \"type\": \"qr\",\n" +
-            "        \"rollup\": {\n" +
-            "            \n" +
-            "        }\n" +
-            "    },\n" +
-            "    \"tags\": [\n" +
-            "        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-            "    ],\n" +
-            "    \"syncts\": 1550501698819,\n" +
-            "    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-            "    \"flags\": {\n" +
-            "        \"tv_processed\": true,\n" +
-            "        \"dd_processed\": true,\n" +
-            "        \"ldata_retrieved\": false\n" +
-            "    },\n" +
-            "    \"type\": \"events\",\n" +
-            "    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-            "    \"ldata\": {\n" +
-            "        \"country_code\": \"\",\n" +
-            "        \"country\": \"\",\n" +
-            "        \"city\": \"\",\n" +
-            "        \"state\": \"\",\n" +
-            "        \"state_code\": \"\"\n" +
-            "    }  \n" +
-            "}";
-
-    public static final String EVENT_WITHOUT_EID = "{\n" +
-			"    \"@timestamp\": \"2020-06-14T18:31:18.487Z\",\n" +
-			"    \"@version\": \"1\",\n" +
-			"    \"context\": {\n" +
-			"        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-			"    },\n" +
-			"    \"message\": \"478097.fdedf464-72ca-45ae-8100-9a92fac0c46b\\\",\\\"actor\\\":{\\\"id\\\":\\\"org.ekstep.learning.platform\\\",\\\"type\\\":\\\"System\\\"},\\\"context\\\":{\\\"channel\\\":\\\"in.ekstep\\\",\\\"pdata\\\":{\\\"id\\\":\\\"prod.ntp.learning.platform\\\",\\\"pid\\\":\\\"search-service\\\",\\\"ver\\\":\\\"1.0\\\"},\\\"env\\\":\\\"search\\\"},\\\"edata\\\":{\\\"size\\\":0,\\\"query\\\":\\\"\\\",\\\"filters\\\":{\\\"status\\\":[\\\"Live\\\",\\\"Draft\\\",\\\"Review\\\"],\\\"contentType\\\":[\\\"Resource\\\",\\\"Collection\\\"],\\\"createdFor\\\":\\\"01241408242723225614\\\",\\\"createdOn\\\":{\\\">=\\\":\\\"2020-06-08T00:00:00.000+0000\\\",\\\"<\\\":\\\"2020-06-15T00:00:00.000+0000\\\"}},\\\"sort\\\":{},\\\"type\\\":\\\"all\\\",\\\"topn\\\":[]},\\\"syncts\\\":1592159478097}\",\n" +
-			"    \"tags\": [\n" +
-			"        \"_jsonparsefailure\"\n" +
-			"    ]\n" +
-			"}";
-
-	public static final String EVENT_WITH_NO_VALIDATION_SCHEMA = "{\n" +
-			"    \"actor\": {\n" +
-			"        \"type\": \"User\",\n" +
-			"        \"id\": \"anonymous\"\n" +
-			"    },\n" +
-			"    \"eid\": \"NO_SCHEMA_DEFINED\",\n" +
-			"    \"edata\": {\n" +
-			"        \"visits\": [\n" +
-			"            \n" +
-			"        ],\n" +
-			"        \"type\": \"view\",\n" +
-			"        \"pageid\": \"dialcode\",\n" +
-			"        \"subtype\": \"pageexit\",\n" +
-			"        \"uri\": \"https://play.diksha.gov.in/dialpage/index.html?dialcode=977D3I\"\n" +
-			"    },\n" +
-			"    \"ver\": \"3.0\",\n" +
-			"    \"ets\": 1570817279146,\n" +
-			"    \"context\": {\n" +
-			"        \"uid\": \"anonymous\",\n" +
-			"        \"pdata\": {\n" +
-			"            \"ver\": \"1.7.1\",\n" +
-			"            \"pid\": \"sunbird-portal\",\n" +
-			"            \"id\": \"dev.sunbird.portal\"\n" +
-			"        },\n" +
-			"        \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-			"        \"env\": \"public\",\n" +
-			"        \"did\": \"a49cfadff97c698c1766c71a42779d4e\",\n" +
-			"        \"sid\": \"5fd1cea0-3a9e-11e9-bed5-2f34fab96d07\",\n" +
-			"        \"cdata\": [\n" +
-			"            \n" +
-			"        ],\n" +
-			"        \"rollup\": {\n" +
-			"            \"l1\": \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-			"        }\n" +
-			"    },\n" +
-			"    \"mid\": \"IMPRESSION:bfd4026a4099370da57e3519cd3368c0\",\n" +
-			"    \"object\": {\n" +
-			"        \"ver\": \"1.0\",\n" +
-			"        \"id\": \"977d3i\",\n" +
-			"        \"type\": \"dialcode\",\n" +
-			"        \"rollup\": {\n" +
-			"            \n" +
-			"        }\n" +
-			"    },\n" +
-			"    \"tags\": [\n" +
-			"        \"505c7c48ac6dc1edc9b08f21db5a571d\"\n" +
-			"    ],\n" +
-			"    \"syncts\": 1550501698819,\n" +
-			"    \"@timestamp\": \"2019-02-18T14:54:58.819Z\",\n" +
-			"    \"flags\": {\n" +
-			"        \"tv_processed\": true,\n" +
-			"        \"dd_processed\": true,\n" +
-			"        \"ldata_retrieved\": false\n" +
-			"    },\n" +
-			"    \"type\": \"events\",\n" +
-			"    \"ts\": \"2019-02-27T14:45:51.866+0000\",\n" +
-			"    \"ldata\": {\n" +
-			"        \"country_code\": \"\",\n" +
-			"        \"country\": \"\",\n" +
-			"        \"city\": \"\",\n" +
-			"        \"state\": \"\",\n" +
-			"        \"state_code\": \"\"\n" +
-			"    }  \n" +
-			"}";
-
-	public static final String EVENT_WITH_DIFFERENT_SCHEMA_VERSION = "{\"actor\":{\"type\":\"User\",\"id\":\"4ba2bb28-16c6-483d-9d94-58c6364e5e3c\"},\"eid\":\"ASSESS\",\"edata\":{\"resvalues\":[{\"3\":{\"text\":\"24n\"}}],\"duration\":21.0,\"score\":0.0,\"item\":{\"mmc\":[],\"mc\":[],\"exlength\":0.0,\"id\":\"do_31296568773394432011706\",\"maxscore\":1.0,\"type\":\"mcq\",\"params\":[{\"1\":{\"text\":\"22n\"}},{\"2\":{\"text\":\"23n\"}},{\"answer\":{\"correct\":[\"4\"]}}],\"title\":\"Test title\",\"uri\":\"\",\"desc\":\"\"},\"pass\":\"No\",\"index\":1.0},\"ver\":\"3.1\",\"ets\":1.592982195439E12,\"context\":{\"pdata\":{\"ver\":\"3.0.0\",\"pid\":\"sunbird-portal.contentplayer\",\"id\":\"prod.diksha.portal\"},\"channel\":\"0126684405014528002\",\"env\":\"contentplayer\",\"did\":\"6798fa5a2d8335c43ba64d5b96a944b9\",\"sid\":\"lRc_DKH5C4034jqv8q1R4O3jxBACvd7e\",\"cdata\":[{\"type\":\"course\",\"id\":\"do_31296569862596198411604\"},{\"type\":\"batch\",\"id\":\"0129657011429376000\"},{\"type\":\"Source\",\"id\":[\"0126087586736619522\",\"0125196274181898243\"]}],\"rollup\":{\"l1\":\"0126684405014528002\"}},\"mid\":\"ASSESS:4ca1ce884b8eeb679abe8ec58c0ac763\",\"object\":{\"ver\":\"1\",\"id\":\"do_3129656882081382401299\",\"type\":\"Content\",\"rollup\":{\"l1\":\"do_3129655631697100801195\",\"l2\":\"do_31296569862596198411604\",\"l3\":\"do_3129656882081382401299\"}},\"tags\":[\"0126684405014528002\"],\"syncts\":1.592982225289E12,\"@timestamp\":\"2020-06-24T07:03:45.289Z\",\"flags\":{\"tv_skipped\":true,\"dd_processed\":true},\"type\":\"events\"}";
-
-	public static final String ANY_STRING = "Hey Samza, Whats Up?";
-	public static final String EMPTY_JSON = "{}";
-
-}
diff --git a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java b/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java
deleted file mode 100644
index 907499ef1b..0000000000
--- a/data-pipeline/telemetry-validator/src/test/java/org/ekstep/ep/samza/task/TelemetryValidatorTaskTest.java
+++ /dev/null
@@ -1,362 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-
-import com.google.common.reflect.TypeToken;
-import com.google.gson.Gson;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.OutgoingMessageEnvelope;
-import org.apache.samza.system.SystemStream;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.fixtures.EventFixture;
-import org.ekstep.ep.samza.util.TelemetrySchemaValidator;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentMatcher;
-
-import java.lang.reflect.Type;
-import java.util.Map;
-
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.*;
-
-public class TelemetryValidatorTaskTest {
-
-    private static final String SUCCESS_TOPIC = "telemetry.valid";
-    private static final String FAILED_TOPIC = "telemetry.failed";
-    private static final String MALFORMED_TOPIC = "telemetry.malformed";
-    private static final String SCHEMA_PATH = "src/test/resources";
-    private static final String SCHEMA_VERSION = "3.0";
-    private MessageCollector collectorMock;
-    private TaskCoordinator coordinatorMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private TelemetryValidatorTask telemetryValidatorTask;
-
-    @Before
-    public void setUp() throws Exception {
-        collectorMock = mock(MessageCollector.class);
-        TaskContext contextMock = mock(TaskContext.class);
-        MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
-
-        Counter counter = mock(Counter.class);
-        coordinatorMock = mock(TaskCoordinator.class);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        Config configMock = mock(Config.class);
-
-        stub(configMock.get("output.success.topic.name", SUCCESS_TOPIC)).toReturn(SUCCESS_TOPIC);
-        stub(configMock.get("output.failed.topic.name", FAILED_TOPIC)).toReturn(FAILED_TOPIC);
-        stub(configMock.get("output.malformed.topic.name", MALFORMED_TOPIC)).toReturn(MALFORMED_TOPIC);
-        stub(configMock.get("telemetry.schema.path", "/etc/samza-jobs/schemas")).toReturn(SCHEMA_PATH);
-        stub(configMock.get("telemetry.schema.version", "3.0")).toReturn(SCHEMA_VERSION);
-        TelemetrySchemaValidator telemetrySchemaValidator = new TelemetrySchemaValidator(new TelemetryValidatorConfig(configMock));
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-        stub(envelopeMock.getOffset()).toReturn("2");
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "input.topic", new Partition(1)));
-
-
-        telemetryValidatorTask = new TelemetryValidatorTask(configMock, contextMock, telemetrySchemaValidator);
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicIfEventIsValid() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_IMPRESSION_EVENT);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    @Test
-    public void shouldSendEventToFailedTopicIfSchemaIsNotPresent() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_NO_VALIDATION_SCHEMA);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                Map<String, Object> metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType);
-                assertEquals(false, flags.get("tv_processed"));
-                assertEquals("SCHEMA NOT FOUND", metadata.get("tv_error"));
-                assertEquals("TelemetryValidator", metadata.get("src"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventToMalformedTopicIfEventIsNotParseable() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.UNPARSEABLE_EVENT);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-    }
-
-    @Test
-    public void shouldSendEventToMalformedTopicIfEventIsAnyRandomString() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), MALFORMED_TOPIC)));
-    }
-
-    @Test
-    public void shouldSendEventToSuccessTopicIfEventIsEmptyJSON() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EMPTY_JSON);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    public void shouldSendEventToFailedTopicIfEidNotPresent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_EID_MISSING);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-    }
-
-    @Test
-    public void shouldRemoveFederatedUserIdPrefixIfPresent() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_IMPRESSION_EVENT);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> actorData = new Gson().fromJson(new Gson().toJson(outputEvent.get("actor")), mapType);
-                assertEquals("874ed8a5-782e-4f6c-8f36-e0288455901e", actorData.get("id"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void whenInvalidEventisPassed() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.INVALID_EVENT);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                assertEquals(false, flags.get("tv_processed"));
-                return true;
-            }
-        }));
-    }
-
-    /**
-     * When valid log event is passed then it should push to success topic
-     *
-     * @throws Exception
-     */
-    @Test
-    public void shouldAddTheValidEventToSuccessTopic() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_EVENT);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-    }
-
-    /**
-     * When channel field is not present in the context then it should push the event to failed topic
-     * @throws Exception
-     */
-    @Test
-    public void shouldValidateTheChannelField() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_WITH_MISSING_CHANNEL);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                assertEquals(false, flags.get("tv_processed"));
-                return true;
-            }
-        }));
-    }
-
-    /**
-     * When synctc and @timestamp fields are missed, Then job should add these fileds to event.
-     * @throws Exception
-     */
-    @Test
-    public void shouldUpdateTheDefaultsToValidEvent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.VALID_LOG_WITH_MISSING_SYNCTS);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), SUCCESS_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                assertEquals(true, flags.get("tv_processed"));
-                assertEquals(true, outputEvent.containsKey("@timestamp"));
-                assertEquals(true, outputEvent.containsKey("syncts"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldConvertDialcodesKeytoLowerCaseIfPresent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.SEARCH_EVENT_WITH_INCORRECT_DIALCODES_KEY);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> edata = new Gson().fromJson(new Gson().toJson(outputEvent.get("edata")), mapType);
-                Map<String, Object> edataFilters = new Gson().fromJson(new Gson().toJson(edata.get("filters")), mapType);
-                assertTrue(edataFilters.containsKey("dialcodes"));
-                assertFalse(edataFilters.containsKey("dialCodes"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldConvertDialcodeValueToUpperCaseIfPresentInLowerCase() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_LOWERCASE);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> object = new Gson().fromJson(new Gson().toJson(outputEvent.get("object")), mapType);
-                assertFalse(object.get("id").toString().equals("977d3i"));
-                assertTrue(object.get("id").toString().equals("977D3I"));
-                return true;
-            }
-        }));
-    } @Test
-    public void shouldNotConvertDialcodeValueToUpperCaseIfNotPresentInLowerCase() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.IMPRESSION_EVENT_WITH_DIALCODE_OBJECT_IN_UPPERCASE);
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> object = new Gson().fromJson(new Gson().toJson(outputEvent.get("object")), mapType);
-                assertTrue(object.get("id").toString().equals("977D3I"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldSendEventToFailedTopicIfEidMissing() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITHOUT_EID);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                Map<String, Object> metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType);
-                assertEquals(false, flags.get("tv_processed"));
-                assertEquals("VALIDATION FAILED as eid is missing", metadata.get("tv_error"));
-                assertEquals("TelemetryValidator", metadata.get("src"));
-                return true;
-            }
-        }));
-    }
-
-    @Test
-    public void shouldValidateAllEventVersions() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.EVENT_WITH_DIFFERENT_SCHEMA_VERSION);
-
-        telemetryValidatorTask.process(envelopeMock, collectorMock, coordinatorMock);
-        verify(collectorMock).send(argThat(validateOutputTopic(envelopeMock.getMessage(), FAILED_TOPIC)));
-        Type mapType = new TypeToken<Map<String, Object>>() {
-        }.getType();
-        verify(collectorMock).send(argThat(new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                String outputMessage = (String) outgoingMessageEnvelope.getMessage();
-                Map<String, Object> outputEvent = new Gson().fromJson(outputMessage, mapType);
-                Map<String, Object> flags = new Gson().fromJson(new Gson().toJson(outputEvent.get("flags")), mapType);
-                Map<String, Object> metadata = new Gson().fromJson(new Gson().toJson(outputEvent.get("metadata")), mapType);
-                assertEquals(false, flags.get("tv_processed"));
-                assertEquals("validation failed. fieldname: \"/context/cdata/2/id\"", metadata.get("tv_error"));
-                assertEquals("TelemetryValidator", metadata.get("src"));
-                return true;
-            }
-        }));
-    }
-
-
-
-    public ArgumentMatcher<OutgoingMessageEnvelope> validateOutputTopic(final Object message, final String stream) {
-        return new ArgumentMatcher<OutgoingMessageEnvelope>() {
-            @Override
-            public boolean matches(Object o) {
-                OutgoingMessageEnvelope outgoingMessageEnvelope = (OutgoingMessageEnvelope) o;
-                SystemStream systemStream = outgoingMessageEnvelope.getSystemStream();
-                assertEquals("kafka", systemStream.getSystem());
-                assertEquals(stream, systemStream.getStream());
-                return true;
-            }
-        };
-    }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json
deleted file mode 100644
index cd87b3fb22..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/actor.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/actor/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/actor/type",
-      "type": "string"
-    }
-  },
-  "required": [ "type", "id"]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json
deleted file mode 100644
index a570db5bc3..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/assess.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/assess",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "ASSESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "item",
-            "pass",
-            "score",
-            "resvalues",
-            "duration"
-          ],
-          "properties": {
-            "item": {
-              "$ref": "resource:/3.0/question.json"
-            },
-            "index": {
-              "id": "http://api.ekstep.org/telemetry/edata/index",
-              "type": "number"
-            },
-            "pass": {
-              "id": "http://api.ekstep.org/telemetry/edata/pass",
-              "type": "string"
-            },
-            "score": {
-              "id": "http://api.ekstep.org/telemetry/edata/score",
-              "type": "number"
-            },
-            "resvalues": {
-              "id": "http://api.ekstep.org/telemetry/edata/resvalues",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json
deleted file mode 100644
index 8dd50aaf7b..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/audit.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/audit",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "AUDIT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "props": {
-              "id": "http://api.ekstep.org/telemetry/edata/props",
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "state": {
-              "id": "http://api.ekstep.org/telemetry/edata/state",
-              "type": "string"
-            },
-            "prevstate": {
-              "id": "http://api.ekstep.org/telemetry/edata/prevstate",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json
deleted file mode 100644
index fd82b2b534..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json
deleted file mode 100644
index 83caf03dfa..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/common.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "3.0"
-      ]
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "actor": {
-      "$ref": "resource:/3.0/actor.json"
-    },
-    "context": {
-      "$ref": "resource:/3.0/context.json"
-    },
-    "object": {
-      "$ref": "resource:/3.0/object.json"
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json
deleted file mode 100644
index 95ac537a49..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/context.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/context/channel",
-      "type": "string",
-      "minLength": 1
-    },
-    "pdata": {
-      "$ref": "resource:/3.0/pdata.json"
-    },
-    "env": {
-      "id": "http://api.ekstep.org/telemetry/context/env",
-      "type": "string"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/context/sid",
-      "type": "string"
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/context/did",
-      "type": "string"
-    },
-    "cdata": {
-      "$ref": "resource:/3.0/cdata.json"
-    }
-  },
-  "required": [
-      "channel",
-      "env",
-      "pdata"
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json
deleted file mode 100644
index cc9ee16701..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/dspec.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "os": {
-      "id": "http://api.ekstep.org/telemetry/dspec/os",
-      "type": "string"
-    },
-    "make": {
-      "id": "http://api.ekstep.org/telemetry/dspec/make",
-      "type": "string"
-    },
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/dspec/id",
-      "type": "string"
-    },
-    "mem": {
-      "id": "http://api.ekstep.org/telemetry/dspec/mem",
-      "type": "number",
-      "minimum": -1
-    },
-    "idisk": {
-      "id": "http://api.ekstep.org/telemetry/dspec/idisk",
-      "type": "number",
-      "minimum": -1
-    },
-    "edisk": {
-      "id": "http://api.ekstep.org/telemetry/dspec/edisk",
-      "type": "number",
-      "minimum": -1
-    },
-    "scrn": {
-      "id": "http://api.ekstep.org/telemetry/dspec/scrn",
-      "type": "number",
-      "minimum": -1
-    },
-    "camera": {
-      "id": "http://api.ekstep.org/telemetry/dspec/camera",
-      "type": "string"
-    },
-    "cpu": {
-      "id": "http://api.ekstep.org/telemetry/dspec/cpu",
-      "type": "string"
-    },
-    "sims": {
-      "id": "http://api.ekstep.org/telemetry/dspec/sims",
-      "type": "number",
-      "minimum": -1
-    },
-    "cap": {
-      "id": "http://api.ekstep.org/telemetry/dspec/cap",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json
deleted file mode 100644
index 298797cdd1..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/end.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "mode": {
-              "id": "http://api.ekstep.org/telemetry/edata/mode",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "summary": {
-              "id": "http://api.ekstep.org/telemetry/edata/summary",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json
deleted file mode 100644
index 8d19e8565c..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/envelope.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "id":"http://api.ekstep.org/telemetry/envelope",
-  "type":"object",
-  "required":[
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf":[
-    {
-      "$ref":"resource:/3.0/common.json"
-    },
-    {
-      "properties":{
-        "eid":{
-          "id":"http://api.ekstep.org/telemetry/mid",
-          "type":"string",
-          "minLength": 1
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json
deleted file mode 100644
index 45e808829f..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/error.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "err",
-            "errtype",
-            "stacktrace"
-          ],
-          "properties": {
-            "err": {
-              "id": "http://api.ekstep.org/telemetry/edata/err",
-              "type": "string"
-            },
-            "errtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/errtype",
-              "type": "string"
-            },
-            "stacktrace": {
-              "id": "http://api.ekstep.org/telemetry/edata/stacktrace",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "object": {
-              "$ref": "resource:/3.0/inlineobject.json"
-            },
-            "plugin": {
-              "$ref": "resource:/3.0/plugin.json"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json
deleted file mode 100644
index eab9d5bbf5..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/event.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [{
-      "$ref": "resource:/3.0/start.json"
-    },
-    {
-      "$ref": "resource:/3.0/end.json"
-    },
-    {
-      "$ref": "resource:/3.0/impression.json"
-    },
-    {
-      "$ref": "resource:/3.0/interact.json"
-    },
-    {
-      "$ref": "resource:/3.0/assess.json"
-    },
-    {
-      "$ref": "resource:/3.0/response.json"
-    },
-    {
-      "$ref": "resource:/3.0/interrupt.json"
-    },
-    {
-      "$ref": "resource:/3.0/feedback.json"
-    },
-    {
-      "$ref": "resource:/3.0/share.json"
-    },
-    {
-      "$ref": "resource:/3.0/audit.json"
-    },
-    {
-      "$ref": "resource:/3.0/error.json"
-    },
-    {
-      "$ref": "resource:/3.0/heartbeat.json"
-    },
-    {
-      "$ref": "resource:/3.0/log.json"
-    },
-    {
-      "$ref": "resource:/3.0/search.json"
-    },
-    {
-      "$ref": "resource:/3.0/exdata.json"
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json
deleted file mode 100644
index 22495c4672..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/exdata.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/exdata",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "EXDATA"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "data": {
-              "id": "http://api.ekstep.org/telemetry/edata/data",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json
deleted file mode 100644
index fa3385f93e..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/feedback.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/feedback",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "rating": {
-              "id": "http://api.ekstep.org/telemetry/edata/rating",
-              "type": "number"
-            },
-            "comments": {
-              "id": "http://api.ekstep.org/telemetry/edata/comments",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json
deleted file mode 100644
index e561482850..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/heartbeat.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/heartbeat",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "HEARTBEAT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object"
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json
deleted file mode 100644
index 44b37a08d2..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/impression.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/impression",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "IMPRESSION"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "pageid",
-            "uri"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "subtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/subtype",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-             },
-            "uri": {
-              "id": "http://api.ekstep.org/telemetry/edata/uri",
-              "type": "string"
-            },
-            "visits": {
-              "id": "http://api.ekstep.org/telemetry/edata/visits",
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "objid": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objid",
-                    "type": "string"
-                  },
-                  "objtype": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objtype",
-                    "type": "string"
-                  },
-                  "objver": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objver",
-                    "type": "string"
-                  },
-                  "section": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/section",
-                    "type": "string"
-                  },
-                  "index": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/index",
-                    "type": "number"
-                  }
-                },
-                "additionalProperties": false,
-                "required": [
-                  "objid",
-                  "objtype"
-                ]
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json
deleted file mode 100644
index 355b8c7834..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/inlineobject.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "type",
-    "ver"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/type",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/ver",
-      "type": "string"
-    },
-    "subtype": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/subtype",
-      "type": "string"
-    },
-    "name": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/name",
-      "type": "string"
-    },
-    "code": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/code",
-      "type": "string"
-    },
-    "parent": {
-      "$ref": "resource:/3.0/parent.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json
deleted file mode 100644
index 4ee513c522..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/interact.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "id"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "subtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/subtype",
-              "type": "string"
-            },
-            "id": {
-              "id": "http://api.ekstep.org/telemetry/edata/id",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "target": {
-              "$ref": "resource:/3.0/target.json"
-            },
-            "plugin": {
-              "$ref": "resource:/3.0/plugin.json"
-            },
-            "extra": {
-              "id": "http://api.ekstep.org/telemetry/edata/extra",
-              "type": "object"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json
deleted file mode 100644
index 2e6b602b89..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/interrupt.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json
deleted file mode 100644
index 937cd76555..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/items.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/items",
-  "type": "array",
-  "items": {
-    "type": "object",
-    "additionalProperties": false,
-    "properties": {
-      "id": {
-        "id": "http://api.ekstep.org/telemetry/items/id",
-        "type": "string"
-      },
-      "type": {
-        "id": "http://api.ekstep.org/telemetry/items/type",
-        "type": "string"
-      },
-      "ver": {
-        "id": "http://api.ekstep.org/telemetry/items/ver",
-        "type": "string"
-      },
-      "params": {
-        "id": "http://api.ekstep.org/telemetry/items/params",
-        "type": "array",
-        "items": {
-          "type": "object"
-        }
-      },
-      "origin": {
-        "id": "http://api.ekstep.org/telemetry/items/origin",
-        "type": "object",
-        "properties": {
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/items/origin/id",
-            "type": "string"
-          },
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/items/origin/type",
-            "type": "string"
-          }
-        }
-      },
-      "to": {
-        "id": "http://api.ekstep.org/telemetry/items/to",
-        "type": "object",
-        "properties": {
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/items/to/id",
-            "type": "string"
-          },
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/items/to/type",
-            "type": "string"
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json
deleted file mode 100644
index 72ebf5af95..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/log.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/log",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "LOG"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "level",
-            "message"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "level": {
-              "id": "http://api.ekstep.org/telemetry/edata/level",
-              "type": "string"
-            },
-            "message": {
-              "id": "http://api.ekstep.org/telemetry/edata/message",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "params": {
-              "id": "http://api.ekstep.org/telemetry/edata/params",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json
deleted file mode 100644
index a22bf39f09..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/object.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/object/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/object/type",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/object/ver",
-      "type": "string"
-    },
-    "rollup": {
-      "$ref": "resource:/3.0/rollup.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json
deleted file mode 100644
index 07d4262af7..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/parent.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/parent/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/parent/type",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json
deleted file mode 100644
index 18f9ba13b6..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/pdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/pdata/id",
-      "type": "string"
-    },
-    "pid": {
-      "id": "http://api.ekstep.org/telemetry/pdata/pid",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/pdata/ver",
-      "type": "string"
-    }
-  },
-  "required": ["id"]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json
deleted file mode 100644
index cfd7d80ba9..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/plugin.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "ver"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/plugin/id",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/plugin/ver",
-      "type": "string"
-    },
-    "category": {
-      "id": "http://api.ekstep.org/telemetry/plugin/category",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json
deleted file mode 100644
index f9ef60f3a0..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/question.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/question/id",
-      "type": "string"
-    },
-    "maxscore": {
-      "id": "http://api.ekstep.org/telemetry/question/maxscore",
-      "type": "number"
-    },
-    "exlength": {
-      "id": "http://api.ekstep.org/telemetry/question/exlength",
-      "type": "number"
-    },
-    "params": {
-      "id": "http://api.ekstep.org/telemetry/question/params",
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "uri": {
-      "id": "http://api.ekstep.org/telemetry/question/uri",
-      "type": "string"
-    },
-    "desc": {
-      "id": "http://api.ekstep.org/telemetry/question/desc",
-      "type": "string"
-    },
-    "title": {
-      "id": "http://api.ekstep.org/telemetry/question/title",
-      "type": "string"
-    },
-    "mmc": {
-      "id": "http://api.ekstep.org/telemetry/question/mmc",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    },
-    "mc": {
-      "id": "http://api.ekstep.org/telemetry/question/mc",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json
deleted file mode 100644
index 4d86626635..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/response.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "RESPONSE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "target",
-            "type",
-            "values"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/type",
-              "type": "string"
-            },
-            "target": {
-              "$ref": "resource:/3.0/target.json"
-            },
-            "values": {
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json
deleted file mode 100644
index bfd89209f2..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/rollup.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "l1": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l1",
-      "type": "string"
-    },
-    "l2": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l2",
-      "type": "string"
-    },
-    "l3": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l3",
-      "type": "string"
-    },
-    "l4": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l4",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json
deleted file mode 100644
index 9bbfc3c3e9..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/search.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/search",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "SEARCH"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "query",
-            "size",
-            "topn"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "query": {
-              "id": "http://api.ekstep.org/telemetry/edata/query",
-              "type": "string"
-            },
-            "filters": {
-              "id": "http://api.ekstep.org/telemetry/edata/filters",
-              "type": "object"
-            },
-            "sort": {
-              "id": "http://api.ekstep.org/telemetry/edata/sort",
-              "type": "object"
-            },
-            "correlationid": {
-              "id": "http://api.ekstep.org/telemetry/edata/correlationid",
-              "type": "string"
-            },
-            "size": {
-              "id": "http://api.ekstep.org/telemetry/edata/size",
-              "type": "number"
-            },
-            "topn": {
-              "id": "http://api.ekstep.org/telemetry/edata/topn",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json
deleted file mode 100644
index 7a21254ff8..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/share.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/share",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "SHARE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "items"
-          ],
-          "properties": {
-            "dir": {
-              "id": "http://api.ekstep.org/telemetry/edata/dir",
-              "type": "string"
-            },
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "items": {
-              "$ref": "resource:/3.0/items.json"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json
deleted file mode 100644
index 78b8f7694a..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/start.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "resource:/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "dspec": {
-              "$ref": "resource:/3.0/dspec.json"
-            },
-            "uaspec": {
-              "$ref": "resource:/3.0/uaspec.json"
-            },
-            "loc": {
-              "id": "http://api.ekstep.org/telemetry/edata/loc",
-              "type": "string"
-            },
-            "mode": {
-              "id": "http://api.ekstep.org/telemetry/edata/mode",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json
deleted file mode 100644
index 6ababb8322..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/target.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "ver",
-    "type"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/target/id",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/target/ver",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/target/type",
-      "type": "string"
-    },
-    "parent": {
-      "$ref": "resource:/3.0/parent.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json b/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json
deleted file mode 100644
index cc7d9f53c6..0000000000
--- a/data-pipeline/telemetry-validator/src/test/resources/3.0/uaspec.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "agent": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/agent",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/ver",
-      "type": "string"
-    },
-    "system": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/system",
-      "type": "string"
-    },
-    "platform": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/platform",
-      "type": "string"
-    },
-    "raw": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/raw",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/data-pipeline/user-cache-updater/pom.xml b/data-pipeline/user-cache-updater/pom.xml
deleted file mode 100644
index 2c0d1a4f94..0000000000
--- a/data-pipeline/user-cache-updater/pom.xml
+++ /dev/null
@@ -1,169 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <parent>
-        <artifactId>jobs</artifactId>
-        <groupId>org.ekstep.ecosystem</groupId>
-        <version>0.0.1</version>
-    </parent>
-         
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>org.ekstep.ecosystem.jobs</groupId>
-    <artifactId>user-cache-updater</artifactId>
-    <version>0.0.2</version>
-    <name>UserCacheUpdater</name>
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-api</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-log4j</artifactId>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.samza</groupId>
-            <artifactId>samza-shell</artifactId>
-            <classifier>dist</classifier>
-            <type>tgz</type>
-            <version>${samza.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.kafka</groupId>
-            <artifactId>kafka_2.11</artifactId>
-            <version>${kafka.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
-            <version>1.7.15</version>
-        </dependency>
-        <dependency>
-            <groupId>org.codehaus.jackson</groupId>
-            <artifactId>jackson-jaxrs</artifactId>
-            <version>1.9.13</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-hdfs</artifactId>
-            <version>2.4.0</version>
-        </dependency>
-        <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.12</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.mockito</groupId>
-            <artifactId>mockito-core</artifactId>
-            <version>2.0.31-beta</version>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-core</artifactId>
-            <version>0.0.1</version>
-        </dependency>
-        <dependency>
-            <groupId>org.ekstep.ecosystem</groupId>
-            <artifactId>ep-telemetry-reader</artifactId>
-            <version>0.0.3</version>
-        </dependency>
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>2.9.0</version>
-        </dependency>
-        <dependency>
-            <groupId>com.google.code.gson</groupId>
-            <artifactId>gson</artifactId>
-            <version>2.4</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fiftyonred</groupId>
-            <artifactId>mock-jedis</artifactId>
-            <version>0.4.0</version>
-            <type>jar</type>
-            <scope>test</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.google.guava</groupId>
-            <artifactId>guava</artifactId>
-            <version>18.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.jetbrains</groupId>
-            <artifactId>annotations</artifactId>
-            <version>RELEASE</version>
-            <scope>compile</scope>
-        </dependency>
-    </dependencies>
-
-    <build>
-        <plugins>
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-surefire-plugin</artifactId>
-                <version>2.20</version>
-            </plugin>
-
-            <plugin>
-                <groupId>org.jacoco</groupId>
-                <artifactId>jacoco-maven-plugin</artifactId>
-                <version>0.8.4</version>
-                <configuration>
-                    <destFile>${basedir}/target/coverage-reports/jacoco-unit.exec</destFile>
-                    <dataFile>${basedir}/target/coverage-reports/jacoco-unit.exec</dataFile>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>jacoco-initialize</id>
-                        <goals>
-                            <goal>prepare-agent</goal>
-                        </goals>
-                    </execution>
-                    <execution>
-                        <id>post-unit-test</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>report</goal>
-                        </goals>
-                        <configuration>
-                            <outputDirectory>${basedir}/target/coverage-reports/jacoco-ut</outputDirectory>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-
-
-</project>
diff --git a/data-pipeline/user-cache-updater/src/main/assembly/src.xml b/data-pipeline/user-cache-updater/src/main/assembly/src.xml
deleted file mode 100644
index a41bb4514d..0000000000
--- a/data-pipeline/user-cache-updater/src/main/assembly/src.xml
+++ /dev/null
@@ -1,70 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <includeBaseDirectory>false</includeBaseDirectory>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}</directory>
-            <includes>
-                <include>README*</include>
-                <include>LICENSE*</include>
-                <include>NOTICE*</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>${basedir}/src/main/resources/log4j.xml</source>
-            <outputDirectory>lib</outputDirectory>
-        </file>
-        <!-- filtered=true, so we do variable expansion so the yarn package path
-          always points to the correct spot on any machine -->
-        <file>
-            <source>${basedir}/src/main/config/user-cache-updater.properties</source>
-            <outputDirectory>config</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>bin</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-shell:tgz:dist:*</include>
-            </includes>
-            <fileMode>0744</fileMode>
-            <unpack>true</unpack>
-        </dependencySet>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <includes>
-                <include>org.apache.samza:samza-core_2.11</include>
-                <include>org.apache.samza:samza-kafka_2.11</include>
-                <include>org.apache.samza:samza-yarn_2.11</include>
-                <include>org.apache.samza:samza-log4j</include>
-                <include>org.ekstep.ecosystem.jobs:user-cache-updater</include>
-                <include>org.slf4j:slf4j-log4j12</include>
-                <include>org.apache.kafka:kafka_2.11</include>
-                <include>org.apache.hadoop:hadoop-hdfs</include>
-                <include>redis.clients:2.9.0</include>
-            </includes>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties b/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties
deleted file mode 100644
index a0d6556cdd..0000000000
--- a/data-pipeline/user-cache-updater/src/main/config/user-cache-updater.properties
+++ /dev/null
@@ -1,96 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Job
-job.factory.class=org.apache.samza.job.yarn.YarnJobFactory
-job.name=__env__.UserCacheUpdater
-
-# YARN
-#yarn.package.path=file://${basedir}/target/${project.artifactId}-${pom.version}-distribution.tar.gz
-yarn.package.path=http://__yarn_host__:__yarn_port__/__env__/${project.artifactId}-${pom.version}-distribution.tar.gz
-#yarn.package.path=http://172.31.12.23:8000/${project.artifactId}-${pom.version}-distribution.tar.gz
-job.container.count=__redis_updater_yarn_container_count__
-
-# Task
-task.class=org.ekstep.ep.samza.task.UserCacheUpdaterTask
-task.inputs=kafka.__env__.telemetry.audit
-task.checkpoint.factory=org.apache.samza.checkpoint.kafka.KafkaCheckpointManagerFactory
-task.checkpoint.system=kafka
-task.checkpoint.replication.factor=__samza_checkpoint_replication_factor__
-task.commit.ms=60000
-task.window.ms=300000
-
-# Systems
-systems.kafka.samza.factory=org.apache.samza.system.kafka.KafkaSystemFactory
-systems.kafka.samza.key.serde=string
-systems.kafka.samza.msg.serde=string
-# systems.kafka.consumer.zookeeper.connect=localhost:2181/
-systems.kafka.consumer.zookeeper.connect=__zookeepers__
-systems.kafka.consumer.auto.offset.reset=smallest
-systems.kafka.samza.offset.default=oldest
-systems.kafka.producer.bootstrap.servers=__kafka_brokers__
-systems.kafka.streams.__env__.metrics.samza.msg.serde=metrics
-systems.kafka.consumer.fetch.message.max.bytes=__consumer_fetch_max_bytes__
-systems.kafka.producer.max.request.size=__producer_max_request_size_bytes__
-
-
-# Metrics
-metrics.reporters=snapshot
-metrics.reporter.snapshot.class=org.apache.samza.metrics.reporter.MetricsSnapshotReporterFactory
-metrics.reporter.snapshot.stream=kafka.__env__.metrics
-output.metrics.topic.name=__env__.pipeline_metrics
-output.malformed.topic.name=__env__.telemetry.malformed
-output.prometheus.metrics.topic.name=__env__.prom.monitoring.metrics
-pipeline.metrics.list=success-message-count,skipped-message-count,error-message-count
-
-# Job Coordinator
-job.coordinator.system=kafka
-job.coordinator.replication.factor=__samza_checkpoint_replication_factor__
-
-# Serializers
-serializers.registry.json.class=org.apache.samza.serializers.JsonSerdeFactory
-serializers.registry.string.class=org.apache.samza.serializers.StringSerdeFactory
-serializers.registry.metrics.class=org.apache.samza.serializers.MetricsSnapshotSerdeFactory
-
-
-# redis
-# redis.host=localhost
-redis.host=__metadata_redis_host__
-# redis.port=6379
-redis.port=__metadata_redis_port__
-redis.connection.max=3
-redis.connection.idle.max=3
-redis.connection.idle.min=1
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.database.userStore.id=4
-
-# content model fields to convert to List from String
-contentModel.fields.listType=gradeLevel,subject,medium,language
-user.signin.type.default = __user_sigin_type_default__
-user.login.type.default = __user_login_type_default__
-user.selfsignedin.typeList = google,self
-user.validated.typeList = sso
-user.self-siginin.key = Self-Signed-In
-user.valid.key = Validated
-
-# sunbird middleware cassandra config
-middleware.cassandra.host=__middleware_cassandra_host__
-middleware.cassandra.port=__middleware_cassandra_port__
-middleware.cassandra.keyspace=__middleware_cassandra_keyspace__
-middleware.cassandra.user_table=__middleware_cassandra_user_table__
-middleware.cassandra.location_table=__middleware_cassandra_location_table__
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java
deleted file mode 100644
index aa9ad22d9d..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/domain/Event.java
+++ /dev/null
@@ -1,71 +0,0 @@
-package org.ekstep.ep.samza.domain;
-
-import org.ekstep.ep.samza.reader.NullableValue;
-import org.ekstep.ep.samza.reader.Telemetry;
-
-import java.util.ArrayList;
-import java.util.Map;
-
-public class Event {
-    private final Telemetry telemetry;
-
-    public Event(Map<String, Object> map) {
-        this.telemetry = new Telemetry(map);
-    }
-
-    public String objectUserId() {
-        NullableValue<String> objectType = telemetry.read("object.type");
-        NullableValue<String> objectUserId = null;
-        if (null != objectType.value() && objectType.value().equalsIgnoreCase("User")) {
-            objectUserId = telemetry.read("object.id");
-        }
-        return null!=objectUserId ? objectUserId.value(): null;
-    }
-
-    public String getUserSignInType() {
-        NullableValue<ArrayList<Map<String, String>>> cdata = telemetry.read("context.cdata");
-        ArrayList<Map<String, String>> cdataList = cdata.value();
-        if (null != cdataList && !cdataList.isEmpty()) {
-            for (Map<String, String> cdataMap : cdataList) {
-                if (cdataMap.containsKey("type") && cdataMap.get("type").equalsIgnoreCase("SignupType"))
-                    return cdataMap.get("id").toString();
-            }
-        }
-        return null;
-    }
-
-    public String getUserLoginType() {
-        NullableValue<ArrayList<Map<String, String>>> cdata = telemetry.read("context.cdata");
-        ArrayList<Map<String, String>> cdataList = cdata.value();
-        if (null != cdataList && !cdataList.isEmpty()) {
-            for (Map<String, String> cdataMap : cdataList) {
-                if (cdataMap.containsKey("type") && cdataMap.get("type").equalsIgnoreCase("UserRole"))
-                    return cdataMap.get("id").toString();
-            }
-        }
-        return null;
-    }
-
-    public ArrayList<String> getUserMetdataUpdatedList() {
-        NullableValue<ArrayList<String>> edata_props = telemetry.read("edata.props");
-        if(null != edata_props) {
-            ArrayList<String> edata_props_list = edata_props.value();
-            if(null != edata_props_list && !edata_props_list.isEmpty()) {
-                return edata_props_list;
-            }
-        }
-        return null;
-    }
-
-    public String getUserStateValue() {
-        NullableValue<String> edata_state = telemetry.read("edata.state");
-        if( null != edata_state.value() && edata_state.value().equalsIgnoreCase("Create") || edata_state.value().equalsIgnoreCase("Created")) {
-            return "Create";
-        }
-        else if( null != edata_state.value() && edata_state.value().equalsIgnoreCase("Update") || edata_state.value().equalsIgnoreCase("Updated")) {
-            return "Update";
-        }
-        return null;
-    }
-
-}
\ No newline at end of file
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java
deleted file mode 100644
index 0a3ea9ca9e..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/service/UserCacheUpdaterService.java
+++ /dev/null
@@ -1,150 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.datastax.driver.core.ColumnDefinitions;
-import com.datastax.driver.core.Row;
-import com.datastax.driver.core.querybuilder.Clause;
-import com.datastax.driver.core.querybuilder.QueryBuilder;
-import com.google.gson.Gson;
-import com.google.gson.JsonSyntaxException;
-import com.google.gson.reflect.TypeToken;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.core.Logger;
-import org.ekstep.ep.samza.domain.Event;
-import org.ekstep.ep.samza.task.UserCacheUpdaterSink;
-import org.ekstep.ep.samza.task.UserCacheUpdaterSource;
-import org.ekstep.ep.samza.task.UserCacheUpdaterConfig;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.ekstep.ep.samza.core.BaseCacheUpdaterService;
-
-import java.lang.reflect.Type;
-import java.util.*;
-
-public class UserCacheUpdaterService extends BaseCacheUpdaterService {
-
-    private static Logger LOGGER = new Logger(UserCacheUpdaterService.class);
-    private JobMetrics metrics;
-    private UserCacheUpdaterConfig userCacheUpdaterConfig;
-    private int userStoreDb;
-    private Gson gson = new Gson();
-    private Type mapType = new TypeToken<Map<String, Object>>() {
-    }.getType();
-
-    public UserCacheUpdaterService(UserCacheUpdaterConfig config, RedisConnect redisConnect, CassandraConnect cassandraConnect, JobMetrics metrics) {
-        super(redisConnect, cassandraConnect);
-        this.metrics = metrics;
-        userCacheUpdaterConfig = config;
-        this.userStoreDb = config.userStoreDb();
-    }
-
-    public void process(UserCacheUpdaterSource source, UserCacheUpdaterSink sink) {
-        try {
-            Event event = source.getEvent();
-            String userId = event.objectUserId();
-            if (null != userId ) {
-                updateUserCache(event, userId, sink);
-            } else { 
-            	sink.markSkipped(); 
-            }
-        } catch (JsonSyntaxException e) {
-        	e.printStackTrace();
-            LOGGER.error(null, "INVALID EVENT: " + source.getMessage());
-            sink.error();
-        }
-    }
-
-    public Map<String, Object> updateUserCache(Event event, String userId, UserCacheUpdaterSink sink) {
-        Map<String, Object> userCacheData = new HashMap<>();
-        String userState = event.getUserStateValue();
-        if("Update".equals(userState)) {
-            String data = readFromCache(userId, userStoreDb);
-            if (data != null && !data.isEmpty()) {
-                userCacheData = gson.fromJson(data, mapType);
-            }
-            userCacheData.putAll(getUserLoginType(event));
-            ArrayList<String> userUpdatedList = event.getUserMetdataUpdatedList();
-            if (userCacheData.containsKey("usersignintype") && !"Anonymous".equals(userCacheData.get("usersignintype"))
-                    && null != userUpdatedList && !userUpdatedList.isEmpty()) {
-                Clause userDataClause = QueryBuilder.eq("id",userId);
-                List<Row> userDetails = readFromCassandra(userCacheUpdaterConfig.cassandra_db(), userCacheUpdaterConfig.cassandra_user_table(), userDataClause);
-                Map<String, Object> userMetadataInfoMap = getUserMetaDataInfo(userDetails);
-
-                if (!userMetadataInfoMap.isEmpty())
-                    userCacheData.putAll(userMetadataInfoMap);
-
-                if (userUpdatedList.contains("id") && null != userCacheData.get("locationids")) {
-                    List<String> locationIds = (List<String>) userCacheData.get("locationids");
-                    Clause locationDataClause = QueryBuilder.in("id",locationIds);
-
-                    List<Row> locationDetails = readFromCassandra(userCacheUpdaterConfig.cassandra_db(), userCacheUpdaterConfig.cassandra_location_table(), locationDataClause);
-                    Map<String, Object> userLocationMap = getLocationInfo(locationDetails);
-
-                    if (!userLocationMap.isEmpty())
-                        userCacheData.putAll(userLocationMap);
-                }
-            }
-        }
-        else if("Create".equals(userState)) {
-            userCacheData.putAll(getUserSigninType(event));
-        }
-
-        if (!userCacheData.isEmpty()) {
-            addToCache(userId, gson.toJson(userCacheData), userStoreDb);
-            LOGGER.info(userId, "Updated in cache");
-            sink.success();
-        } else {
-        	sink.markSkipped();
-        }
-        return userCacheData;
-    }
-
-    private Map<String, String> getUserSigninType(Event event) {
-        Map<String,String> userData = new HashMap<>();
-        String signInType = null != event.getUserSignInType() ? event.getUserSignInType() : userCacheUpdaterConfig.getUserSignInTypeDefault();
-        if (!(userCacheUpdaterConfig.getUserSignInTypeDefault().equalsIgnoreCase(signInType))) {
-            if (userCacheUpdaterConfig.getUserSelfSignedInTypeList().contains(signInType)) {
-                userData.put("usersignintype", userCacheUpdaterConfig.getUserSelfSignedKey());
-            } else if (userCacheUpdaterConfig.getUserValidatedTypeList().contains(signInType)) {
-                userData.put("usersignintype", userCacheUpdaterConfig.getUserValidatedKey());
-            }
-        }
-        return userData;
-    }
-
-    private Map<String, String> getUserLoginType(Event event) {
-        Map<String,String> userData = new HashMap<>();
-        String loginInType = null != event.getUserLoginType() ? event.getUserLoginType() : userCacheUpdaterConfig.getUserLoginInTypeDefault();
-        if (!(userCacheUpdaterConfig.getUserLoginInTypeDefault().equalsIgnoreCase(loginInType)))
-            userData.put("userlogintype", loginInType);
-        return userData;
-    }
-
-    public Map<String, Object> getUserMetaDataInfo(List<Row> userDetails) {
-        Map<String, Object> result = new HashMap<>();
-        if( null != userDetails && userDetails.size() > 0) {
-            Row row= userDetails.get(0);
-            ColumnDefinitions columnDefinitions = row.getColumnDefinitions();
-            int columnCount = columnDefinitions.size();
-            for(int i=0; i<columnCount;i++){
-                result.put(columnDefinitions.getName(i),row.getObject(i));
-            }
-        }
-        return result;
-    }
-
-    public Map<String, Object> getLocationInfo(List<Row> locationDetails) {
-        Map<String, Object> result = new HashMap<>();
-        if(null != locationDetails && !locationDetails.isEmpty()) {
-            locationDetails.forEach(record -> {
-                String name = record.getString("name");
-                String type = record.getString("type");
-                if(type.toLowerCase().equals("state")) {
-                    result.put("state", name);
-                } else if(type.toLowerCase().equals("district")) {
-                    result.put("district", name);
-                }
-            });
-        }
-        return result;
-    }
-}
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java
deleted file mode 100644
index d8351b5cb1..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterConfig.java
+++ /dev/null
@@ -1,58 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.config.Config;
-
-import java.util.Arrays;
-import java.util.List;
-
-public class UserCacheUpdaterConfig {
-
-    private final String JOB_NAME = "UserCacheUpdater";
-
-    private String userSignInTypeDefault;
-    private String userLoginInTypeDefault;
-    private List<String> userSelfSignedInTypeList;
-    private List<String> userValidatedTypeList;
-    private String userSelfSignedKey;
-    private String userValidatedKey;
-    private String cassandra_db;
-    private String cassandra_user_table;
-    private String cassandra_location_table;
-    private int userStoreDb;
-
-    public UserCacheUpdaterConfig(Config config) {
-        this.userSignInTypeDefault = config.get("user.signin.type.default", "Anonymous");
-        this.userLoginInTypeDefault = config.get("user.login.type.default", "NA");
-        this.userSelfSignedInTypeList = config.getList("user.selfsignedin.typeList", Arrays.asList("google", "self"));
-        this.userValidatedTypeList = config.getList("user.validated.typeList", Arrays.asList("sso"));
-        this.userSelfSignedKey = config.get("user.self-siginin.key", "Self-Signed-In");
-        this.userValidatedKey = config.get("user.valid.key", "Validated");
-        this.cassandra_db = config.get("middleware.cassandra.keyspace", "sunbird");
-        this.cassandra_user_table = config.get("middleware.cassandra.user_table", "user");
-        this.cassandra_location_table = config.get("middleware.cassandra.location_table", "location");
-        this.userStoreDb = config.getInt("redis.database.userStore.id", 4);
-    }
-
-    public String getUserSignInTypeDefault() { return userSignInTypeDefault; }
-
-    public String getUserLoginInTypeDefault() { return userLoginInTypeDefault; }
-
-    public String getUserSelfSignedKey() { return userSelfSignedKey; }
-
-    public String getUserValidatedKey() { return userValidatedKey; }
-
-    public List<String> getUserSelfSignedInTypeList() { return userSelfSignedInTypeList; }
-
-    public List<String> getUserValidatedTypeList() { return userValidatedTypeList; }
-
-    public String cassandra_db() { return cassandra_db; }
-
-    public String cassandra_user_table() { return cassandra_user_table; }
-
-    public String cassandra_location_table() { return cassandra_location_table; }
-
-    public String jobName() { return JOB_NAME; }
-
-    public int userStoreDb() { return userStoreDb; }
-
-}
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java
deleted file mode 100644
index 02e612da99..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSink.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import org.apache.samza.task.MessageCollector;
-import org.ekstep.ep.samza.core.BaseSink;
-import org.ekstep.ep.samza.core.JobMetrics;
-
-public class UserCacheUpdaterSink extends BaseSink {
-
-    public UserCacheUpdaterSink(MessageCollector collector, JobMetrics metrics) {
-        super(collector, metrics);
-    }
-
-    public void success() {
-        metrics.incSuccessCounter();
-    }
-
-    public void error() {
-        metrics.incErrorCounter();
-    }
-
-    public void markSkipped() { metrics.incSkippedCounter(); }
-
-}
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java
deleted file mode 100644
index 1e8817f40e..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterSource.java
+++ /dev/null
@@ -1,27 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import com.google.gson.Gson;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.ekstep.ep.samza.domain.Event;
-
-import java.util.Map;
-
-public class UserCacheUpdaterSource {
-
-    private IncomingMessageEnvelope envelope;
-
-    public UserCacheUpdaterSource(IncomingMessageEnvelope envelope) {
-        this.envelope = envelope;
-    }
-
-    public Event getEvent() {
-        String message = (String) envelope.getMessage();
-        @SuppressWarnings("unchecked")
-        Map<String, Object> jsonMap = (Map<String, Object>) new Gson().fromJson(message, Map.class);
-        return new Event(jsonMap);
-    }
-
-    public String getMessage() {
-        return envelope.toString();
-    }
-}
diff --git a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java b/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java
deleted file mode 100644
index 773145260d..0000000000
--- a/data-pipeline/user-cache-updater/src/main/java/org/ekstep/ep/samza/task/UserCacheUpdaterTask.java
+++ /dev/null
@@ -1,53 +0,0 @@
-package org.ekstep.ep.samza.task;
-
-import java.util.Arrays;
-import java.util.List;
-
-import org.apache.samza.config.Config;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.UserCacheUpdaterService;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-
-public class UserCacheUpdaterTask extends BaseSamzaTask {
-
-    private JobMetrics metrics;
-    private UserCacheUpdaterConfig config;
-    private UserCacheUpdaterService service;
-
-    public UserCacheUpdaterTask(Config config, TaskContext context, CassandraConnect cassandraConnect, RedisConnect redisConnect) {
-        init(config, context, cassandraConnect, redisConnect);
-    }
-
-    public UserCacheUpdaterTask() {
-    }
-
-    @Override
-    public void init(Config config, TaskContext context) {
-        init(config, context, null, null);
-    }
-
-    public void init(Config config, TaskContext context, CassandraConnect cassandraConnect, RedisConnect redisConnect) {
-        this.config = new UserCacheUpdaterConfig(config);
-        metrics = new JobMetrics(context, this.config.jobName());
-        List<String> cassandraHosts = Arrays.asList(config.get("middleware.cassandra.host", "127.0.0.1").split(","));
-        cassandraConnect = null != cassandraConnect ? cassandraConnect : new CassandraConnect(cassandraHosts, config.getInt("middleware..cassandra.port", 9042));
-        redisConnect = null != redisConnect? redisConnect: new RedisConnect(config);
-        service = new UserCacheUpdaterService(this.config, redisConnect, cassandraConnect, metrics);
-        this.initTask(config, metrics);
-    }
-
-    @Override
-    public void process(IncomingMessageEnvelope envelope, MessageCollector collector,
-                        TaskCoordinator taskCoordinator) throws Exception {
-
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelope);
-        UserCacheUpdaterSink sink = new UserCacheUpdaterSink(collector, metrics);
-        service.process(source, sink);
-    }
-
-}
diff --git a/data-pipeline/user-cache-updater/src/main/resources/log4j.xml b/data-pipeline/user-cache-updater/src/main/resources/log4j.xml
deleted file mode 100644
index 7e0a688c9b..0000000000
--- a/data-pipeline/user-cache-updater/src/main/resources/log4j.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
-
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-
--->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
-    <appender name="jmx" class="org.apache.samza.logging.log4j.JmxAppender" />
-
-    <appender name="RollingAppender" class="org.apache.log4j.RollingFileAppender">
-        <param name="File" value="${samza.log.dir}/${samza.container.name}.log" />
-        <param name="append" value="true" />
-        <param name="maxFileSize" value="100MB" />
-        <param name="maxBackupIndex" value="5" />
-        <layout class="org.apache.log4j.PatternLayout">
-            <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss} %c{1} [%p] %m%n" />
-        </layout>
-    </appender>
-    <root>
-        <priority value="info" />
-        <appender-ref ref="RollingAppender"/>
-        <appender-ref ref="jmx" />
-    </root>
-</log4j:configuration>
diff --git a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java b/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
deleted file mode 100644
index 5e86146f4a..0000000000
--- a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/Fixtures/EventFixture.java
+++ /dev/null
@@ -1,404 +0,0 @@
-package org.ekstep.ep.samza.service.Fixtures;
-
-public class EventFixture {
-
-    public static final String AUDIT_EVENT = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606dfdd6\",\"actor\":{\"id\":\"627a431d-4f5c-4adc-812d-1f01c5588555\",\"type\":\"User\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"cdata\":[{\"id\":\"25cb0530-7c52-ecb1-cff2-6a14faab7910\",\"type\":\"SignupType\"}],\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"id\":\"627a431d-4f5c-4adc-812d-1f01c5588555\",\"type\":\"User\"},\"edata\":{\"state\":\"Update\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}";
-    public static final String AUDIT_EVENT2 = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606df\",\"actor\":{\"id\":\"5609876543234567890987654345678\",\"type\":\"Request\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"cdata\":[{\"id\":\"25cb0530-7c52-ecb1-cff2-6a14faab7910\",\"type\":\"SignupType\"}],\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"id\":\"5609876543h2fd34h5678jf909876af54345678\"},\"edata\":{\"state\":\"Update\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}";
-    public static final String AUDIT_EVENT3 = "{\"eid\":\"AUDIT\",\"ets\":1.573121861118E12,\"ver\":\"3.0\",\"mid\":\"1573121861118.40f9136b-1cc3-458d-a04a-4459606df\",\"actor\":{\"id\":\"5609876543234567890987654345678\",\"type\":\"Request\"},\"context\":{\"channel\":\"01285019302823526477\",\"pdata\":{\"id\":\"dev.sunbird.portal\",\"pid\":\"learner-service\",\"ver\":\"2.5.0\"},\"env\":\"User\",\"did\":\"2bcfc645e27e64625f7bad6ce282f9d0\",\"rollup\":{\"l1\":\"01285019302823526477\"}},\"object\":{\"type\":\"User\",\"id\":\"5609876543h2fd34h5678jf909876af54345678\"},\"edata\":{\"state\":\"Create\",\"props\":[\"recoveryEmail\",\"recoveryPhone\",\"userId\",\"id\",\"externalIds\",\"updatedDate\",\"updatedBy\"]},\"syncts\":1.573121861125E12,\"@timestamp\":\"2019-11-07T10:17:41.125Z\",\"flags\":{\"tv_processed\":true,\"dd_processed\":true},\"type\":\"events\",\"ts\":\"2019-11-07T10:17:41.118+0000\"}";
-
-    public static final String AUDIT_EVENT_SIGINTYPE="{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"Consumer\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Create\",\n" +
-            "    \"props\": [\n" +
-            "      \"firstName\",\n" +
-            "      \"email\",\n" +
-            "      \"emailVerified\",\n" +
-            "      \"id\",\n" +
-            "      \"userId\",\n" +
-            "      \"createdBy\",\n" +
-            "      \"rootOrgId\",\n" +
-            "      \"channel\",\n" +
-            "      \"userType\",\n" +
-            "      \"roles\",\n" +
-            "      \"phoneVerified\",\n" +
-            "      \"isDeleted\",\n" +
-            "      \"createdDate\",\n" +
-            "      \"status\",\n" +
-            "      \"userName\",\n" +
-            "      \"loginId\",\n" +
-            "      \"externalIds\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561739226844,\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"0126684405014528002\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"learner-service\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"id\": \"prod.diksha.learning.service\"\n" +
-            "    },\n" +
-            "    \"env\": \"User\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"SignupType\",\n" +
-            "        \"id\": \"google\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Source\",\n" +
-            "        \"id\": \"android\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Request\",\n" +
-            "        \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"0126684405014528002\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" +
-            "  \"object\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"syncts\": 1561739243532,\n" +
-            "  \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String AUDIT_EVENT_SIGIN_TYPE="{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"Consumer\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Created\",\n" +
-            "    \"props\": [\n" +
-            "      \"firstName\",\n" +
-            "      \"email\",\n" +
-            "      \"emailVerified\",\n" +
-            "      \"id\",\n" +
-            "      \"userId\",\n" +
-            "      \"createdBy\",\n" +
-            "      \"rootOrgId\",\n" +
-            "      \"channel\",\n" +
-            "      \"userType\",\n" +
-            "      \"roles\",\n" +
-            "      \"phoneVerified\",\n" +
-            "      \"isDeleted\",\n" +
-            "      \"createdDate\",\n" +
-            "      \"status\",\n" +
-            "      \"userName\",\n" +
-            "      \"loginId\",\n" +
-            "      \"externalIds\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561739226844,\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"0126684405014528002\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"learner-service\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"id\": \"prod.diksha.learning.service\"\n" +
-            "    },\n" +
-            "    \"env\": \"User\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"SignupType\",\n" +
-            "        \"id\": \"sso\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Source\",\n" +
-            "        \"id\": \"android\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Request\",\n" +
-            "        \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"0126684405014528002\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" +
-            "  \"object\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"syncts\": 1561739243532,\n" +
-            "  \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String AUDIT_EVENT_NULL_USERID ="{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"Consumer\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Create\",\n" +
-            "    \"props\": [\n" +
-            "      \"firstName\",\n" +
-            "      \"email\",\n" +
-            "      \"emailVerified\",\n" +
-            "      \"id\",\n" +
-            "      \"userId\",\n" +
-            "      \"createdBy\",\n" +
-            "      \"rootOrgId\",\n" +
-            "      \"channel\",\n" +
-            "      \"userType\",\n" +
-            "      \"roles\",\n" +
-            "      \"phoneVerified\",\n" +
-            "      \"isDeleted\",\n" +
-            "      \"createdDate\",\n" +
-            "      \"status\",\n" +
-            "      \"userName\",\n" +
-            "      \"loginId\",\n" +
-            "      \"externalIds\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561739226844,\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"0126684405014528002\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"learner-service\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"id\": \"prod.diksha.learning.service\"\n" +
-            "    },\n" +
-            "    \"env\": \"User\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"SignupType\",\n" +
-            "        \"id\": \"sso\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Source\",\n" +
-            "        \"id\": \"android\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Request\",\n" +
-            "        \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"0126684405014528002\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" +
-            "  \"object\": {\n" +
-            "    \"type\": \"User\"\n" +
-            "  },\n" +
-            "  \"syncts\": 1561739243532,\n" +
-            "  \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String AUDIT_EVENT_SIGN_IN="{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"Consumer\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Create\",\n" +
-            "    \"props\": [\n" +
-            "      \"firstName\",\n" +
-            "      \"email\",\n" +
-            "      \"emailVerified\",\n" +
-            "      \"id\",\n" +
-            "      \"userId\",\n" +
-            "      \"createdBy\",\n" +
-            "      \"rootOrgId\",\n" +
-            "      \"channel\",\n" +
-            "      \"userType\",\n" +
-            "      \"roles\",\n" +
-            "      \"phoneVerified\",\n" +
-            "      \"isDeleted\",\n" +
-            "      \"createdDate\",\n" +
-            "      \"status\",\n" +
-            "      \"userName\",\n" +
-            "      \"loginId\",\n" +
-            "      \"externalIds\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561739226844,\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"0126684405014528002\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"learner-service\",\n" +
-            "      \"ver\": \"2.0.0\",\n" +
-            "      \"id\": \"prod.diksha.learning.service\"\n" +
-            "    },\n" +
-            "    \"env\": \"User\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"User\",\n" +
-            "        \"id\": \"34881c3a-8b92-4a3c-a982-7f946137cb09\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"SignupType\",\n" +
-            "        \"id\": \"sso\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Source\",\n" +
-            "        \"id\": \"android\"\n" +
-            "      },\n" +
-            "      {\n" +
-            "        \"type\": \"Request\",\n" +
-            "        \"id\": \"91f3c280-99c1-11e9-956e-6b6ef71ed575\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"0126684405014528002\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"mid\": \"1561739226844.e0048ef8-a01e-4780-8c83-e571f28c53c8\",\n" +
-            "  \"object\": {\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"id\": \"89490534-126f-4f0b-82ac-3ff3e49f3468\"\n" +
-            "  },\n" +
-            "  \"syncts\": 1561739243532,\n" +
-            "  \"@timestamp\": \"2019-06-28T16:27:23.532Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-    public static final String AUDIT_EVENT_LOGININTYPE ="{\n" +
-            "  \"actor\": {\n" +
-            "    \"type\": \"System\",\n" +
-            "    \"id\": \"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d\"\n" +
-            "  },\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Update\",\n" +
-            "    \"props\": [\n" +
-            "      \"medium\",\n" +
-            "      \"board\",\n" +
-            "      \"grade\",\n" +
-            "      \"syllabus\",\n" +
-            "      \"gradeValue\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"ets\": 1561739240727,\n" +
-            "  \"context\": {\n" +
-            "    \"pdata\": {\n" +
-            "      \"pid\": \"sunbird.app\",\n" +
-            "      \"ver\": \"2.1.92\",\n" +
-            "      \"id\": \"prod.diksha.app\"\n" +
-            "    },\n" +
-            "    \"channel\": \"505c7c48ac6dc1edc9b08f21db5a571d\",\n" +
-            "    \"env\": \"sdk\",\n" +
-            "    \"did\": \"010612971a80a7677d0a3e849ab35cb4a83157de\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"type\": \"UserRole\",\n" +
-            "        \"id\": \"student\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"sid\": \"ea68a05e-0843-4c06-9a84-9b98cd974724\"\n" +
-            "  },\n" +
-            "  \"mid\": \"0268860e-76b0-4b4e-b99b-ebf543e7a9d8\",\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"3b46b4c9-3a10-439a-a2cb-feb5435b3a0d\",\n" +
-            "    \"type\": \"User\",\n" +
-            "    \"version\": \"\",\n" +
-            "    \"rollup\": {}\n" +
-            "  },\n" +
-            "  \"syncts\": 1561739245463,\n" +
-            "  \"@timestamp\": \"2019-06-28T16:27:25.463Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\"\n" +
-            "}";
-
-    public static final String AUDIT_EVENT_METADATA_UPDATED = "{\n" +
-            "  \"eid\": \"AUDIT\",\n" +
-            "  \"ets\": 1571297660511,\n" +
-            "  \"ver\": \"3.0\",\n" +
-            "  \"mid\": \"1571297660511.32f5024a-aa30-4c82-abd8-bb8d8914ed2d\",\n" +
-            "  \"actor\": {\n" +
-            "    \"id\": \"ef70da5a-bb99-4785-b970-1d6d6ee75aad\",\n" +
-            "    \"type\": \"User\"\n" +
-            "  },\n" +
-            "  \"context\": {\n" +
-            "    \"channel\": \"01285019302823526477\",\n" +
-            "    \"pdata\": {\n" +
-            "      \"id\": \"dev.sunbird.portal\",\n" +
-            "      \"pid\": \"learner-service\",\n" +
-            "      \"ver\": \"2.4.0\"\n" +
-            "    },\n" +
-            "    \"env\": \"User\",\n" +
-            "    \"did\": \"bfaf115f65a2086b062735885f4d2f1a\",\n" +
-            "    \"cdata\": [\n" +
-            "      {\n" +
-            "        \"id\": \"1b1392bc-39d8-6e47-d7d6-5781a2f1481a\",\n" +
-            "        \"type\": \"Request\"\n" +
-            "      }\n" +
-            "    ],\n" +
-            "    \"rollup\": {\n" +
-            "      \"l1\": \"01285019302823526477\"\n" +
-            "    }\n" +
-            "  },\n" +
-            "  \"object\": {\n" +
-            "    \"id\": \"52226956-61d8-4c1b-b115-c660111866d3\",\n" +
-            "    \"type\": \"User\"\n" +
-            "  },\n" +
-            "  \"edata\": {\n" +
-            "    \"state\": \"Updated\",\n" +
-            "    \"props\": [\n" +
-            "      \"firstName\",\n" +
-            "      \"userId\",\n" +
-            "      \"id\",\n" +
-            "      \"externalIds\",\n" +
-            "      \"locationIds\",\n" +
-            "      \"updatedDate\",\n" +
-            "      \"updatedBy\"\n" +
-            "    ]\n" +
-            "  },\n" +
-            "  \"syncts\": 1571297660521,\n" +
-            "  \"@timestamp\": \"2019-10-17T07:34:20.521Z\",\n" +
-            "  \"flags\": {\n" +
-            "    \"tv_processed\": true,\n" +
-            "    \"dd_processed\": true\n" +
-            "  },\n" +
-            "  \"type\": \"events\",\n" +
-            "  \"ts\": \"2019-10-17T07:34:20.511+0000\"\n" +
-            "}";
-
-    public static final String ANY_STRING = "Any String";
-}
diff --git a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java b/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java
deleted file mode 100644
index b08e26eb68..0000000000
--- a/data-pipeline/user-cache-updater/src/test/java/org/ekstep/ep/samza/service/UserCacheUpdaterServiceTest.java
+++ /dev/null
@@ -1,285 +0,0 @@
-package org.ekstep.ep.samza.service;
-
-import com.datastax.driver.core.Row;
-import com.fiftyonred.mock_jedis.MockJedis;
-import com.google.gson.Gson;
-import com.google.gson.reflect.TypeToken;
-import org.apache.samza.Partition;
-import org.apache.samza.config.Config;
-import org.apache.samza.metrics.Counter;
-import org.apache.samza.metrics.MetricsRegistry;
-import org.apache.samza.system.IncomingMessageEnvelope;
-import org.apache.samza.system.SystemStreamPartition;
-import org.apache.samza.task.MessageCollector;
-import org.apache.samza.task.TaskContext;
-import org.apache.samza.task.TaskCoordinator;
-import org.ekstep.ep.samza.core.JobMetrics;
-import org.ekstep.ep.samza.service.Fixtures.EventFixture;
-import org.ekstep.ep.samza.task.UserCacheUpdaterConfig;
-import org.ekstep.ep.samza.task.UserCacheUpdaterSink;
-import org.ekstep.ep.samza.task.UserCacheUpdaterSource;
-import org.ekstep.ep.samza.task.UserCacheUpdaterTask;
-import org.ekstep.ep.samza.core.BaseCacheUpdaterService;
-import org.ekstep.ep.samza.util.CassandraConnect;
-import org.ekstep.ep.samza.util.RedisConnect;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.Mockito;
-import redis.clients.jedis.Jedis;
-
-import java.lang.reflect.Type;
-import java.util.*;
-
-import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.*;
-import static org.mockito.Mockito.times;
-
-public class UserCacheUpdaterServiceTest {
-
-    private RedisConnect redisConnectMock;
-    private CassandraConnect cassandraConnectMock;
-    private IncomingMessageEnvelope envelopeMock;
-    private TaskContext contextMock;
-    private JobMetrics jobMetricsMock;
-    private TaskCoordinator taskCoordinator;
-    private MessageCollector messageCollector;
-    private Jedis jedisMock = new MockJedis("test");
-    private UserCacheUpdaterService userCacheUpdaterService;
-    private BaseCacheUpdaterService basecacheUpdater;
-    private UserCacheUpdaterSink userCacheUpdaterSinkMock;
-    private UserCacheUpdaterConfig userCacheUpdaterConfig;
-    private MetricsRegistry metricsRegistry;
-    private Counter counter;
-    private Config configMock;
-    private Integer userStoreId = 4;
-    private UserCacheUpdaterTask userCacheUpdaterTask;
-
-    @Before
-    public void setUp() {
-        redisConnectMock = mock(RedisConnect.class);
-        userCacheUpdaterSinkMock = mock(UserCacheUpdaterSink.class);
-        configMock = mock(Config.class);
-        cassandraConnectMock = mock(CassandraConnect.class);
-        contextMock = mock(TaskContext.class);
-        jobMetricsMock = mock(JobMetrics.class);
-        metricsRegistry = Mockito.mock(MetricsRegistry.class);
-        taskCoordinator = mock(TaskCoordinator.class);
-        messageCollector = mock(MessageCollector.class);
-        counter = mock(Counter.class);
-
-        stub(redisConnectMock.getConnection(userStoreId)).toReturn(jedisMock);
-        stub(redisConnectMock.getConnection()).toReturn(jedisMock);
-        envelopeMock = mock(IncomingMessageEnvelope.class);
-        stub(configMock.getInt("redis.database.userStore.id", userStoreId)).toReturn(userStoreId);
-        stub(configMock.getInt("location.db.redis.key.expiry.seconds", 86400)).toReturn(86400);
-        stub(configMock.get("user.signin.type.default", "Anonymous")).toReturn("Anonymous");
-        stub(configMock.getList("user.selfsignedin.typeList", Arrays.asList("google", "self"))).toReturn(Arrays.asList("google", "self"));
-        stub(configMock.getList("user.validated.typeList", Arrays.asList("sso"))).toReturn(Arrays.asList("sso"));
-        stub(configMock.get("user.login.type.default", "NA")).toReturn("NA");
-        stub(configMock.get("user.self-siginin.key", "Self-Signed-In")).toReturn("Self-Signed-In");
-        stub(configMock.get("user.valid.key", "Validated")).toReturn("Validated");
-        stub(configMock.get("middleware.cassandra.host", "127.0.0.1")).toReturn("");
-        stub(configMock.get("middleware.cassandra.port", "9042")).toReturn("9042");
-        stub(configMock.get("middleware.cassandra.keyspace", "sunbird")).toReturn("sunbird");
-        stub(configMock.get("middleware.cassandra.user_table","user")).toReturn("user");
-        stub(configMock.get("middleware.cassandra.location_table","location")).toReturn("location");
-
-        stub(metricsRegistry.newCounter(anyString(), anyString())).toReturn(counter);
-        stub(contextMock.getMetricsRegistry()).toReturn(metricsRegistry);
-
-        userCacheUpdaterConfig=new UserCacheUpdaterConfig(configMock);
-        userCacheUpdaterService = new UserCacheUpdaterService(userCacheUpdaterConfig, redisConnectMock, cassandraConnectMock, jobMetricsMock);
-        jedisMock.flushAll();
-    }
-
-    @Test
-    public void shouldNotUpdateCacheForInvalidEvent() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.ANY_STRING);
-        userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock, redisConnectMock);
-        userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator);
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-
-        verify(userCacheUpdaterSinkMock, times(1)).error();
-    }
-
-    @Test
-    public void shouldAddLocationDetailsToCache() {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED);
-        Gson gson = new Gson();
-        String userId = "52226956-61d8-4c1b-b115-c660111866d3";
-        jedisMock.select(userStoreId);
-        jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false,\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"NA\", \"locationids\":[\'8952478975387\']}");
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(5, parsedData.size());
-        assertEquals("dikshacustodian",parsedData.get("channel"));
-        assertEquals(false, parsedData.get("phoneverified"));
-        assertEquals("Self-Signed-In", parsedData.get("usersignintype"));
-        assertEquals( "NA", parsedData.get("userlogintype"));
-        assertEquals(Arrays.asList("8952478975387"), parsedData.get("locationids"));
-    }
-
-    @Test
-    public void shouldNotUpdateCacheForAnonymousUsers() throws Exception {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT3);
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-    }
-
-    @Test
-    public void shouldMarkEventSkippedForNullUserId() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_NULL_USERID);
-        userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock);
-        userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator);
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-        verify(userCacheUpdaterSinkMock, times(1)).markSkipped();
-    }
-
-    @Test public void shouldupdateLocationDetails() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT);
-        userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock);
-        userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator);
-
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-    }
-
-    @Test
-    public void shouldUpdateUserSignInDetailsinCacheFORAUDIT() {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGINTYPE);
-        Gson gson = new Gson();
-        String userId = "89490534-126f-4f0b-82ac-3ff3e49f3468";
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-        Map<String, Object> userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock);
-
-        jedisMock.set(userId, gson.toJson(userData));
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals("Self-Signed-In", parsedData.get("usersignintype"));
-    }
-
-    @Test
-    public void shouldUpdateUserDetailsinCacheFORAUDIT() {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGN_IN);
-        Gson gson = new Gson();
-        String userId = "89490534-126f-4f0b-82ac-3ff3e49f3468";
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-        Map<String, Object> userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock);
-        jedisMock.set(userId, gson.toJson(userData));
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals("Validated", parsedData.get("usersignintype"));
-    }
-
-    @Test
-    public void shouldUpdateUserLoginInTYpeDetailsinCacheFORAUDIT() {
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_LOGININTYPE);
-        Gson gson = new Gson();
-        String userId = "3b46b4c9-3a10-439a-a2cb-feb5435b3a0d";
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition())
-                .toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-        Map<String, Object> userData = userCacheUpdaterService.updateUserCache(source.getEvent(), userId, userCacheUpdaterSinkMock);
-        jedisMock.set(userId, gson.toJson(userData));
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals("student", parsedData.get("userlogintype"));
-    }
-
-    @Test
-    public void shouldNotUpdateCacheWithMetadataChangesAndLocationFORAUDIT() {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED);
-        Gson gson = new Gson();
-        String userId = "52226956-61d8-4c1b-b115-c660111866d3";
-        jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false}");
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = new HashMap<>();
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(0, parsedData.size());
-        verify(cassandraConnectMock, times(0)).find(anyString());
-    }
-
-    @Test
-    public void shouldUpdateCache() throws Exception {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGINTYPE);
-        userCacheUpdaterTask = new UserCacheUpdaterTask(configMock, contextMock, cassandraConnectMock,redisConnectMock);
-        userCacheUpdaterTask.process(envelopeMock,messageCollector,taskCoordinator);
-
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_SIGIN_TYPE);
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-    }
-
-    @Test
-    public void shouldUpdateCacheWithMetadataChangesAndLocationFORAUDIT() {
-        stub(envelopeMock.getMessage()).toReturn(EventFixture.AUDIT_EVENT_METADATA_UPDATED);
-        Gson gson = new Gson();
-        String userId = "52226956-61d8-4c1b-b115-c660111866d3";
-        jedisMock.select(userStoreId);
-        jedisMock.set(userId, "{\"channel\":\"dikshacustodian\",\"phoneverified\":false,\"usersignintype\":\"Self-Signed-In\",\"userlogintype\":\"NA\"}");
-        UserCacheUpdaterSource source = new UserCacheUpdaterSource(envelopeMock);
-        stub(envelopeMock.getSystemStreamPartition()).toReturn(new SystemStreamPartition("kafka", "telemetry.audit", new Partition(1)));
-        userCacheUpdaterService.process(source, userCacheUpdaterSinkMock);
-
-        String cachedData = jedisMock.get(userId);
-        Map<String, Object> parsedData = null;
-        if (cachedData != null) {
-            Type type = new TypeToken<Map<String, Object>>() {
-            }.getType();
-            parsedData = gson.fromJson(cachedData, type);
-        }
-        assertEquals(4, parsedData.size());
-        assertEquals(parsedData.get("channel"), "dikshacustodian");
-        assertEquals(parsedData.get("phoneverified"), false);
-        assertEquals(parsedData.get("usersignintype"), "Self-Signed-In");
-        assertEquals(parsedData.get("userlogintype"), "NA");
-    }
-}
diff --git a/druid/adhoc-jobs/dependency-reduced-pom.xml b/druid/adhoc-jobs/dependency-reduced-pom.xml
deleted file mode 100644
index 39716f4f57..0000000000
--- a/druid/adhoc-jobs/dependency-reduced-pom.xml
+++ /dev/null
@@ -1,119 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-  <modelVersion>4.0.0</modelVersion>
-  <groupId>org.ekstep</groupId>
-  <artifactId>adhoc-jobs</artifactId>
-  <version>1.0</version>
-  <build>
-    <sourceDirectory>src/main/scala</sourceDirectory>
-    <testSourceDirectory>src/test/scala</testSourceDirectory>
-    <finalName>adhoc-jobs-1.0</finalName>
-    <plugins>
-      <plugin>
-        <groupId>net.alchim31.maven</groupId>
-        <artifactId>scala-maven-plugin</artifactId>
-        <version>3.2.2</version>
-        <executions>
-          <execution>
-            <goals>
-              <goal>compile</goal>
-              <goal>testCompile</goal>
-            </goals>
-            <configuration>
-              <args>
-                <arg>-dependencyfile</arg>
-                <arg>${project.build.directory}/.scala_dependencies</arg>
-              </args>
-            </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <groupId>org.scalatest</groupId>
-        <artifactId>scalatest-maven-plugin</artifactId>
-        <version>2.0.0</version>
-        <executions>
-          <execution>
-            <id>test</id>
-            <phase>test</phase>
-            <goals>
-              <goal>test</goal>
-            </goals>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
-        <artifactId>maven-shade-plugin</artifactId>
-        <version>2.3</version>
-        <executions>
-          <execution>
-            <phase>package</phase>
-            <goals>
-              <goal>shade</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <transformers>
-            <transformer>
-              <resource>META-INF/services/org.jclouds.apis.ApiMetadata</resource>
-            </transformer>
-            <transformer>
-              <resource>META-INF/services/org.jclouds.providers.ProviderMetadata</resource>
-            </transformer>
-          </transformers>
-          <filters>
-            <filter>
-              <artifact>*:*</artifact>
-              <excludes>
-                <artifact>*:*</artifact>
-                <exclude>META-INF/*.SF</exclude>
-                <exclude>META-INF/*.DSA</exclude>
-                <exclude>META-INF/*.RSA</exclude>
-              </excludes>
-            </filter>
-          </filters>
-        </configuration>
-      </plugin>
-      <plugin>
-        <artifactId>maven-assembly-plugin</artifactId>
-        <version>2.3</version>
-        <executions>
-          <execution>
-            <id>make-assembly</id>
-            <phase>package</phase>
-            <goals>
-              <goal>single</goal>
-            </goals>
-          </execution>
-        </executions>
-        <configuration>
-          <descriptors>
-            <descriptor>src/main/assembly/src.xml</descriptor>
-          </descriptors>
-        </configuration>
-      </plugin>
-    </plugins>
-  </build>
-  <dependencies>
-    <dependency>
-      <groupId>org.scala-lang</groupId>
-      <artifactId>scala-library</artifactId>
-      <version>2.11.8</version>
-      <scope>provided</scope>
-    </dependency>
-    <dependency>
-      <groupId>org.scalatest</groupId>
-      <artifactId>scalatest_2.11</artifactId>
-      <version>2.2.4</version>
-      <scope>test</scope>
-    </dependency>
-  </dependencies>
-  <properties>
-    <scala.version>2.11.8</scala.version>
-    <scoverage.plugin.version>1.1.1</scoverage.plugin.version>
-    <scala.maj.version>2.11</scala.maj.version>
-    <spark.version>2.0.1</spark.version>
-  </properties>
-</project>
-
diff --git a/druid/adhoc-jobs/pom.xml b/druid/adhoc-jobs/pom.xml
deleted file mode 100644
index a418f7c4d7..0000000000
--- a/druid/adhoc-jobs/pom.xml
+++ /dev/null
@@ -1,232 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0"
-         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
-    <modelVersion>4.0.0</modelVersion>
-
-    <groupId>org.ekstep</groupId>
-    <artifactId>adhoc-jobs</artifactId>
-    <version>1.0</version>
-
-    <properties>
-        <spark.version>2.4.4</spark.version>
-        <scala.version>2.11.8</scala.version>
-        <scala.maj.version>2.11</scala.maj.version>
-        <scoverage.plugin.version>1.1.1</scoverage.plugin.version>
-    </properties>
-
-    <dependencies>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-core_${scala.maj.version}</artifactId>
-            <version>${spark.version}</version>
-            <scope>provided</scope>
-            <exclusions>
-                <exclusion>
-                    <artifactId>jets3t</artifactId>
-                    <groupId>net.java.dev.jets3t</groupId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.scala-lang</groupId>
-            <artifactId>scala-library</artifactId>
-            <version>${scala.version}</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-annotations</artifactId>
-            <version>2.7.8</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-core</artifactId>
-            <version>2.7.8</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.7.8</version>
-        </dependency>
-        <dependency>
-            <groupId>com.fasterxml.jackson.module</groupId>
-            <artifactId>jackson-module-scala_${scala.maj.version}</artifactId>
-            <version>2.7.8</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.commons</groupId>
-            <artifactId>commons-text</artifactId>
-            <version>1.6</version>
-        </dependency>
-        <dependency>
-            <groupId>com.datastax.spark</groupId>
-            <artifactId>spark-cassandra-connector_${scala.maj.version}</artifactId>
-            <version>2.0.8</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-mllib_${scala.maj.version}</artifactId>
-            <version>${spark.version}</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.spark</groupId>
-                    <artifactId>spark-core_2.11</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.scalatest</groupId>
-            <artifactId>scalatest_${scala.maj.version}</artifactId>
-            <version>2.2.4</version>
-            <scope>test</scope>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/com.typesafe/config -->
-        <dependency>
-            <groupId>com.typesafe</groupId>
-            <artifactId>config</artifactId>
-            <version>1.3.3</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch-spark-20 -->
-        <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch-spark-20_2.11</artifactId>
-            <version>6.6.2</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/com.redislabs/spark-redis -->
-        <dependency>
-            <groupId>com.redislabs</groupId>
-            <artifactId>spark-redis</artifactId>
-            <version>2.3.1-RC1</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
-        <dependency>
-            <groupId>redis.clients</groupId>
-            <artifactId>jedis</artifactId>
-            <version>3.0.1</version>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.apache.spark/spark-streaming -->
-        <dependency>
-            <groupId>org.apache.spark</groupId>
-            <artifactId>spark-streaming_2.11</artifactId>
-            <version>2.1.0</version>
-            <scope>provided</scope>
-        </dependency>
-        <!-- https://mvnrepository.com/artifact/org.sunbird/cloud-store-sdk -->
-        <dependency>
-            <groupId>org.sunbird</groupId>
-            <artifactId>cloud-store-sdk</artifactId>
-            <version>1.2.6</version>
-            <exclusions>
-                <exclusion>
-                    <groupId>com.microsoft.azure</groupId>
-                    <artifactId>azure-storage</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.fasterxml.jackson.core</groupId>
-                    <artifactId>jackson-core</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.httpcomponents</groupId>
-                    <artifactId>httpclient</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>com.google.guava</groupId>
-                    <artifactId>guava</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>com.microsoft.azure</groupId>
-            <artifactId>azure-storage</artifactId>
-            <version>3.0.0</version>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-client</artifactId>
-            <version>2.7.3</version>
-            <scope>provided</scope>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-common</artifactId>
-            <version>2.7.3</version>
-            <scope>provided</scope>
-            <exclusions>
-                <exclusion>
-                    <groupId>org.apache.avro</groupId>
-                    <artifactId>avro</artifactId>
-                </exclusion>
-            </exclusions>
-        </dependency>
-        <dependency>
-            <groupId>org.apache.hadoop</groupId>
-            <artifactId>hadoop-aws</artifactId>
-            <version>2.7.3</version>
-            <scope>provided</scope>
-        </dependency>
-    </dependencies>
-
-
-    <build>
-        <finalName>adhoc-jobs-1.0</finalName>
-        <sourceDirectory>src/main/scala</sourceDirectory>
-        <testSourceDirectory>src/test/scala</testSourceDirectory>
-        <plugins>
-            <plugin>
-                <!-- see http://davidb.github.com/scala-maven-plugin -->
-                <groupId>net.alchim31.maven</groupId>
-                <artifactId>scala-maven-plugin</artifactId>
-                <version>3.2.2</version>
-                <executions>
-                    <execution>
-                        <goals>
-                            <goal>compile</goal>
-                            <goal>testCompile</goal>
-                        </goals>
-                        <configuration>
-                            <args>
-                                <arg>-dependencyfile</arg>
-                                <arg>${project.build.directory}/.scala_dependencies</arg>
-                            </args>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-            <plugin>
-                <groupId>org.scalatest</groupId>
-                <artifactId>scalatest-maven-plugin</artifactId>
-                <version>2.0.0</version>
-                <executions>
-                    <execution>
-                        <id>test</id>
-                        <phase>test</phase>
-                        <goals>
-                            <goal>test</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>2.3</version>
-                <configuration>
-                    <descriptors>
-                        <descriptor>src/main/assembly/src.xml</descriptor>
-                    </descriptors>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
-    
-</project>
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/main/assembly/src.xml b/druid/adhoc-jobs/src/main/assembly/src.xml
deleted file mode 100644
index 66dc0a42c8..0000000000
--- a/druid/adhoc-jobs/src/main/assembly/src.xml
+++ /dev/null
@@ -1,57 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
-  license agreements. See the NOTICE file distributed with this work for additional
-  information regarding copyright ownership. The ASF licenses this file to
-  you under the Apache License, Version 2.0 (the "License"); you may not use
-  this file except in compliance with the License. You may obtain a copy of
-  the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
-  by applicable law or agreed to in writing, software distributed under the
-  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
-  OF ANY KIND, either express or implied. See the License for the specific
-  language governing permissions and limitations under the License. -->
-
-<assembly
-        xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2"
-        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.2 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
-    <id>distribution</id>
-    <formats>
-        <format>tar.gz</format>
-    </formats>
-    <fileSets>
-        <fileSet>
-            <directory>${basedir}/src/main/resources</directory>
-            <outputDirectory>/resources</outputDirectory>
-            <includes>
-                <include>*.conf</include>
-            </includes>
-        </fileSet>
-        <fileSet>
-            <directory>${basedir}/src/scripts</directory>
-            <outputDirectory>/scripts</outputDirectory>
-            <includes>
-                <include>*.sh</include>
-            </includes>
-        </fileSet>
-        <fileSet>
-            <directory>target</directory>
-            <outputDirectory>/</outputDirectory>
-            <includes>
-                <include>*.jar</include>
-            </includes>
-        </fileSet>
-    </fileSets>
-    <files>
-        <file>
-            <source>../ingestion-spec/content_index_batch.json</source>
-            <outputDirectory>/druid_models</outputDirectory>
-            <filtered>true</filtered>
-        </file>
-    </files>
-    <dependencySets>
-        <dependencySet>
-            <outputDirectory>lib</outputDirectory>
-            <useTransitiveFiltering>true</useTransitiveFiltering>
-        </dependencySet>
-    </dependencySets>
-</assembly>
diff --git a/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf b/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf
deleted file mode 100644
index 4e6eb88351..0000000000
--- a/druid/adhoc-jobs/src/main/resources/DeviceProfile.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-spark.cassandra.connection.host="__cassandra__host"
-cassandra.keyspace="__env__"
-cassandra.deviceprofileOldTable="device_profile"
-cassandra.deviceprofileNewTable="device_profile_temp"
diff --git a/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf b/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf
deleted file mode 100644
index 9f60abc672..0000000000
--- a/druid/adhoc-jobs/src/main/resources/DialcodeRedisIndexer.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-# redis
-redis.host=__redis_host__
-redis.port="6379"
-redis.connection.max=20
-redis.connection.idle.max=20
-redis.connection.idle.min=10
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.max.pipeline.size="1000"
-redis.dialcode.database.index=6
-
-cloudStorage.accountName=""
-cloudStorage.accountKey=""
-cloudStorage.container="dev-data-store"
-cloudStorage.dialCodeDataFile="dialcode-data/dial_code.csv"
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf b/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf
deleted file mode 100644
index dd1a482c2b..0000000000
--- a/druid/adhoc-jobs/src/main/resources/ESCloudUploader.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-elasticsearch.host="localhost"
-elasticsearch.port="9200"
-elasticsearch.scroll.size="1000"
-elasticsearch.query.index="compositesearch"
-elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}"
-outputFilePath="/home/ops/tmp/compositeSearchBatchData"
-cloudStorage.container="telemetry-data-store"
-cloudStorage.objectKey="druid-content-snapshot/snapshot.txt"
-cloudStorage.provider="azure"
-cloudStorage.accountName=""
-cloudStorage.accountKey=""
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf b/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf
deleted file mode 100644
index a4d7aa897e..0000000000
--- a/druid/adhoc-jobs/src/main/resources/ESContentIndexer.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# redis
-redis.host=__redis_host__
-redis.port="6379"
-redis.connection.max=20
-location.db.redis.key.expiry.seconds=3600
-redis.connection.idle.max=20
-redis.connection.idle.min=10
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.max.pipeline.size="1000"
-
-# EStoRedis config, all values should be in string format
-redis.es.database.index="5"
-# to which Redis database (0 - 15) data should be indexed?
-elasticsearch.host=__es_host__
-elasticsearch.port="9200"
-elasticsearch.scroll.size="1000"
-elasticsearch.index.source.keys=["identifier"] # this will be used as key for redis
-elasticsearch.index.source.keyDelimiter=""
-elasticsearch.query.index="compositesearch"
-# query to fetch all 'Live' status contents of objectType 'Content'
-elasticsearch.query.jsonString="{\"query\":{\"bool\":{\"must\":[{\"match\":{\"status\":{\"query\":\"Live\"}}},{\"match\":{\"objectType\":{\"query\":\"Content\"}}}]}}}"
diff --git a/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf b/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf
deleted file mode 100644
index c24cf9392a..0000000000
--- a/druid/adhoc-jobs/src/main/resources/ESDialcodeIndexer.conf
+++ /dev/null
@@ -1,22 +0,0 @@
-# redis
-redis.host=__redis_host__
-redis.port="6379"
-redis.connection.max=20
-location.db.redis.key.expiry.seconds=3600
-redis.connection.idle.max=20
-redis.connection.idle.min=10
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.max.pipeline.size="1000"
-
-# EStoRedis config, all values should be in string format
-
-redis.es.database.index="6"
-# to which Redis database (0 - 15) data should be indexed?
-elasticsearch.host=__es_host__
-elasticsearch.port="9200"
-elasticsearch.scroll.size="1000"
-elasticsearch.index.source.keys=["identifier"] # this will be used as key for redis
-elasticsearch.index.source.keyDelimiter=""
-elasticsearch.query.index="dialcode"
-elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}"
diff --git a/druid/adhoc-jobs/src/main/resources/application.conf b/druid/adhoc-jobs/src/main/resources/application.conf
deleted file mode 100644
index 216a34592f..0000000000
--- a/druid/adhoc-jobs/src/main/resources/application.conf
+++ /dev/null
@@ -1 +0,0 @@
-# do not delete this file
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf b/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf
deleted file mode 100644
index 3cebe8942b..0000000000
--- a/druid/adhoc-jobs/src/main/resources/cassandraRedis.conf
+++ /dev/null
@@ -1,18 +0,0 @@
-# redis
-redis.host=__redis_host__
-redis.port="6379"
-redis.connection.max=20
-location.db.redis.key.expiry.seconds=3600
-redis.connection.idle.max=20
-redis.connection.idle.min=10
-redis.connection.minEvictableIdleTimeSeconds=120
-redis.connection.timeBetweenEvictionRunsSeconds=300
-redis.max.pipeline.size="1000"
-
-
-#CassandraToRedis Config
-spark.cassandra.connection.host="localhost"
-cassandra.user.keyspace="sunbird"
-cassandra.user.table="user"
-redis.user.database.index="4"
-redis.user.index.source.key="id" # this will be used as key for redis
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala
deleted file mode 100644
index 4cbf94c790..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CSVToRedisIndexer.scala
+++ /dev/null
@@ -1,51 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.spark.{SparkConf, SparkContext}
-import com.redislabs.provider.redis._
-import org.apache.spark.sql.SparkSession
-import org.ekstep.analytics.util.JSONUtils
-
-object CSVToRedisIndexer {
-
-    private val config: Config = ConfigFactory.load
-
-    def main(args: Array[String]): Unit = {
-
-        val container = config.getString("cloudStorage.container")
-        val dialCodeDataFile = config.getString("cloudStorage.dialCodeDataFile")
-        val cloudStorageAccount = config.getString("cloudStorage.accountName")
-
-        val filePath = s"wasbs://$container@$cloudStorageAccount.blob.core.windows.net/$dialCodeDataFile"
-        val conf = new SparkConf()
-          .setAppName("SparkCSVtoRedisIndexer")
-          .setMaster("local[*]")
-
-          // redis settings
-          .set("spark.redis.host", config.getString("redis.host"))
-          .set("spark.redis.port", config.getString("redis.port"))
-          .set("spark.redis.db", config.getString("redis.dialcode.database.index"))
-          .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size"))
-
-        val sc = new SparkContext(conf)
-        val spark = SparkSession.builder.config(conf).getOrCreate()
-
-        spark.sparkContext.hadoopConfiguration.set("fs.azure", "org.apache.hadoop.fs.azure.NativeAzureFileSystem")
-        spark.sparkContext.hadoopConfiguration.set("fs.azure.account.key." + cloudStorageAccount + ".blob.core.windows.net", config.getString("cloudStorage.accountKey"))
-
-        val data = spark.read.option("header", "true").option("inferSchema", "true").csv(filePath).toJSON.rdd.map(f => f)
-        val finalData = data.map(f => (getKey(f), f))
-        finalData.foreach(f => println(f))
-
-        spark.sparkContext.toRedisKV(
-            finalData
-        )
-    }
-
-    def getKey(str: String) ={
-        val dataMap = JSONUtils.deserialize[Map[String, AnyRef]](str)
-        val key = dataMap.get("identifier")
-        key.mkString
-    }
-}
-
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala
deleted file mode 100644
index 74467b39fc..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/CassandraRedisIndexer.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import com.typesafe.config.{Config, ConfigFactory}
-import org.ekstep.analytics.util.JSONUtils
-import com.datastax.spark.connector._
-import com.redislabs.provider.redis._
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkContext
-
-
-object CassandraRedisIndexer {
-
-   private val config: Config = ConfigFactory.load
-
-    def main(args: Array[String]): Unit = {
-
-        val conf = new SparkConf()
-            .setAppName("CassandraToRedisIndexer")
-            .setMaster("local[*]")
-            // Cassandra settings
-            .set("spark.cassandra.connection.host", config.getString("spark.cassandra.connection.host"))
-            // redis settings
-            .set("spark.redis.host", config.getString("redis.host"))
-            .set("spark.redis.port", config.getString("redis.port"))
-            .set("spark.redis.db", config.getString("redis.user.database.index"))
-            .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size"))
-
-        val sc = new SparkContext(conf)
-
-        val userKeyspace = config.getString("cassandra.user.keyspace")
-        val userTableName = config.getString("cassandra.user.table")
-        val redisKeyProperty: String = config.getString("redis.user.index.source.key")
-
-        val userData = sc.cassandraTable(userKeyspace, userTableName)
-        val userMap = userData.map(f => f.toMap)
-        val mappedData = userMap.map { obj =>
-            (obj.getOrElse(redisKeyProperty, "").asInstanceOf[String] , JSONUtils.serialize(obj))
-        }
-        sc.toRedisKV(mappedData)
-    }
-
-
-}
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala
deleted file mode 100644
index b517e6926e..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/DeviceProfileUpdateCassandra.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-package org.ekstep.analytics.jobs
-import org.apache.spark.sql.functions.{row_number}
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.spark.sql.expressions.Window
-import com.datastax.spark.connector._
-import org.apache.spark.sql.SparkSession
-import org.apache.spark.SparkConf
-import org.apache.spark.SparkContext
-import org.apache.spark.sql._
-
-object DeviceProfileUpdateCassandra {
-
-private val config: Config = ConfigFactory.load
-
-def main(args: Array[String]): Unit = {
-
-val conf = new SparkConf()
-            .setAppName("DeviceProfileUpdateCassandra")
-            .setMaster("local[*]")
-            .set("spark.cassandra.connection.host", config.getString("spark.cassandra.connection.host"))
-
-val sc = new SparkContext(conf)
-val spark = SparkSession.builder().master("local[*]").appName("DeviceProfileUpdateCassandra").getOrCreate()
-import spark.implicits._ 
-
-val deviceProfileDf = spark.read.format("org.apache.spark.sql.cassandra").options(Map( "table" -> config.getString("cassandra.deviceprofileOldTable")  , "keyspace" -> config.getString("cassandra.keyspace") )).load()
-
-val windowUpdatedDate = Window.partitionBy($"device_id").orderBy($"updated_date".desc)
-
-val deviceProfileTempDf = deviceProfileDf.filter(deviceProfileDf("state").isNotNull).withColumn("latest_updatedDate", row_number.over(windowUpdatedDate)).where($"latest_updatedDate" === 1).drop("latest_updatedDate","channel")
-
-deviceProfileTempDf.write.format("org.apache.spark.sql.cassandra").options(Map( "table" -> config.getString("cassandra.deviceprofileNewTable"), "keyspace" -> config.getString("cassandra.keyspace") )).mode(SaveMode.Append).save()
-
-}
-
-}
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala
deleted file mode 100644
index d5a138b4ec..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESCloudUploader.scala
+++ /dev/null
@@ -1,54 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import java.io.File
-import java.util.Date
-
-import scala.reflect.io.Directory
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.spark.SparkConf
-import org.apache.spark.sql.SparkSession
-import org.elasticsearch.spark._
-import org.sunbird.cloud.storage.factory.{StorageConfig, StorageServiceFactory}
-
-object ESCloudUploader {
-    private val now: Date = new Date()
-    private val config: Config = ConfigFactory.load
-    case class Source(timestamp: Long, data: String)
-
-    def main(args: Array[String]): Unit = {
-
-        val index = config.getString("elasticsearch.query.index")
-        val query = config.getString("elasticsearch.query.jsonString")
-        val outputFilePath = config.getString("outputFilePath")
-
-        println(s"[$index] query ===> $query")
-
-        require(!index.isEmpty && !query.isEmpty, "require valid inputs! index name and query cannot be empty!")
-
-        val conf = new SparkConf()
-            .setAppName("SparkEStoJSON")
-            .setMaster("local[*]")
-            // Elasticsearch settings
-            .set("es.nodes", config.getString("elasticsearch.host"))
-            .set("es.port", config.getString("elasticsearch.port"))
-            .set("es.scroll.size", config.getString("elasticsearch.scroll.size"))
-            .set("es.query", query)
-
-        val sparkSession = SparkSession.builder.config(conf).getOrCreate
-        val sc = sparkSession.sparkContext
-
-        println(s"deleting output folder if exist! : $outputFilePath")
-        val directory = new Directory(new File(outputFilePath))
-        if (directory.exists) directory.deleteRecursively()
-
-        sc.esJsonRDD(index).map(data => s"""{ "timestamp": ${now.getTime}, "data": ${data._2} }""")
-            .coalesce(1)
-            .saveAsTextFile(outputFilePath)
-
-        // backup the output file to cloud
-        val storageService = StorageServiceFactory.getStorageService(StorageConfig(config.getString("cloudStorage.provider"), config.getString("cloudStorage.accountName"), config.getString("cloudStorage.accountKey")))
-        storageService.upload(config.getString("cloudStorage.container"), outputFilePath + "/part-00000", config.getString("cloudStorage.objectKey"), isDirectory = Option(false))
-        println("successfully backed up file to cloud!")
-        System.exit(0)
-    }
-}
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala
deleted file mode 100644
index 48e4b4a607..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/jobs/ESRedisIndexer.scala
+++ /dev/null
@@ -1,52 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import com.redislabs.provider.redis._
-import com.typesafe.config.{Config, ConfigFactory}
-import org.apache.spark.{SparkConf, SparkContext}
-import org.ekstep.analytics.util.JSONUtils
-import org.elasticsearch.spark._
-import scala.collection.JavaConversions._
-import scala.collection.Map
-
-object ESRedisIndexer {
-
-    private val config: Config = ConfigFactory.load
-
-    def main(args: Array[String]): Unit = {
-
-        val index = config.getString("elasticsearch.query.index")
-        val query = config.getString("elasticsearch.query.jsonString")
-
-        println(s"[$index] query ===> $query")
-
-        require(!index.isEmpty && !query.isEmpty, "require valid inputs! index name and query cannot be empty!")
-
-        val conf = new SparkConf()
-            .setAppName("SparkEStoRedisIndexer")
-            .setMaster("local[*]")
-            // Elasticsearch settings
-            .set("es.nodes", config.getString("elasticsearch.host"))
-            .set("es.port", config.getString("elasticsearch.port"))
-            .set("es.scroll.size", config.getString("elasticsearch.scroll.size"))
-            .set("es.query", query)
-            // redis settings
-            .set("spark.redis.host", config.getString("redis.host"))
-            .set("spark.redis.port", config.getString("redis.port"))
-            .set("spark.redis.db", config.getString("redis.es.database.index"))
-            .set("spark.redis.max.pipeline.size", config.getString("redis.max.pipeline.size"))
-
-        val sc = new SparkContext(conf)
-        val keys = config.getStringList("elasticsearch.index.source.keys").toList
-        val keyDelimiter = config.getString("elasticsearch.index.source.keyDelimiter")
-
-        // todo: log details
-        def getKey(data: String): String = {
-            val record = JSONUtils.deserialize[Map[String, AnyRef]](data)
-            keys.map(value => record(value)).mkString(keyDelimiter)
-        }
-
-        sc.toRedisKV(
-            sc.esJsonRDD(index).map(data => (getKey(data._2), data._2))
-        )
-    }
-}
diff --git a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala b/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala
deleted file mode 100644
index 80857aec0e..0000000000
--- a/druid/adhoc-jobs/src/main/scala/org/ekstep/analytics/util/JSONUtils.scala
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.ekstep.analytics.util
-
-import java.lang.reflect.{ParameterizedType, Type}
-import com.fasterxml.jackson.annotation.JsonInclude.Include
-import com.fasterxml.jackson.core.`type`.TypeReference
-import com.fasterxml.jackson.databind.{DeserializationFeature, ObjectMapper, SerializationFeature}
-import com.fasterxml.jackson.module.scala.DefaultScalaModule
-import org.apache.commons.text.StringEscapeUtils
-
-object JSONUtils {
-
-    @transient val mapper = new ObjectMapper();
-    mapper.registerModule(DefaultScalaModule);
-    mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
-    mapper.configure(SerializationFeature.WRITE_NULL_MAP_VALUES, false);
-    mapper.configure(SerializationFeature.FAIL_ON_EMPTY_BEANS, false);
-    mapper.configure(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS, false);
-    mapper.setSerializationInclusion(Include.NON_NULL);
-
-    @throws(classOf[Exception])
-    def serialize(obj: AnyRef): String = {
-        mapper.writeValueAsString(obj);
-    }
-
-    @throws(classOf[Exception])
-    def deserialize[T: Manifest](value: String): T = mapper.readValue(value, typeReference[T]);
-
-    @throws(classOf[Exception])
-    def unescapeJSON(string: String): String = {
-        StringEscapeUtils.unescapeJava(string)
-    }
-
-    private[this] def typeReference[T: Manifest] = new TypeReference[T] {
-        override def getType = typeFromManifest(manifest[T])
-    }
-
-    
-    private[this] def typeFromManifest(m: Manifest[_]): Type = {
-        if (m.typeArguments.isEmpty) { m.runtimeClass }
-        // $COVERAGE-OFF$Disabling scoverage as this code is impossible to test
-        else new ParameterizedType {
-            def getRawType = m.runtimeClass
-            def getActualTypeArguments = m.typeArguments.map(typeFromManifest).toArray
-            def getOwnerType = null
-        }
-        // $COVERAGE-ON$
-    }
-    
-}
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh b/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh
deleted file mode 100644
index 3e1ab8bc69..0000000000
--- a/druid/adhoc-jobs/src/scripts/DeviceProfileScripts.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-home=`echo $HOME`
-jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar"
-jobConfPath="${home}/adhoc-jobs-1.0/resources/DeviceProfile.conf"
-
-spark/bin/spark-submit \
---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.DeviceProfileUpdateCassandra \
-${jobJarPath}
diff --git a/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh b/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh
deleted file mode 100644
index d43f6a6d2b..0000000000
--- a/druid/adhoc-jobs/src/scripts/DruidContentIndexer.sh
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env bash
-set -e
-# Configurations
-druidCoordinatorIP=""
-dataSourceName="content-model-snapshot"
-today=`date +%Y-%m-%d`
-interval="2019-01-01_$today"
-now=`date +%Y-%m-%d-%s`
-home=`echo $HOME`
-ingestionSpecFilePath="${home}/adhoc-jobs-1.0/druid_models/content_index_batch.json"
-jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar"
-jobConfPath="${home}/adhoc-jobs-1.0/resources/ESCloudUploader.conf"
-
-# get list of segments from content model snapshot datasource
-bkpIFS="$IFS"
-segmentIds=$(curl -X 'GET' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/metadata/datasources/${dataSourceName}/segments)
-IFS=',]['
-read -r -a array <<< ${segmentIds}
-IFS="$bkpIFS"
-
-# start the spark script to fetch Elasticsearch data and write it to a file and upload to cloud
-spark/bin/spark-submit \
---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.ESCloudUploader \
-${jobJarPath}
-
-printf "\n>>> submit ingestion task to Druid!\n"
-
-# submit task to start batch ingestion
-curl -X 'POST' -H 'Content-Type:application/json' -d @${ingestionSpecFilePath} http://${druidCoordinatorIP}:8090/druid/indexer/v1/task
-
-
-for index in "${!array[@]}"
-do
-    val=( $(eval echo ${array[index]}) )
-    printf "\n>>> Disabling segment id: $val \n"
-    # disable older segments
-    curl -X 'DELETE' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/datasources/${dataSourceName}/segments/$val
-done
-
-printf "\n>>> Deleting segments from interval $interval \n"
-
-# delete older segments
-curl -X 'DELETE' -H 'Content-Type:application/json' http://${druidCoordinatorIP}:8081/druid/coordinator/v1/datasources/${dataSourceName}/intervals/${interval}
-
-printf "\n>>> success!\n"
-
-
diff --git a/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh
deleted file mode 100644
index 0e350bcd1b..0000000000
--- a/druid/adhoc-jobs/src/scripts/RedisContentIndexer.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-home=`echo $HOME`
-jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar"
-jobConfPath="${home}/adhoc-jobs-1.0/resources/ESContentIndexer.conf"
-
-spark/bin/spark-submit \
---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.ESRedisIndexer \
-${jobJarPath}
diff --git a/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh
deleted file mode 100755
index b22cd90814..0000000000
--- a/druid/adhoc-jobs/src/scripts/RedisDialcodeIndexer.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-home=`echo $HOME`
-jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar"
-jobConfPath="${home}/adhoc-jobs-1.0/resources/DialcodeRedisIndexer.conf"
-
-spark/bin/spark-submit \
---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.CSVToRedisIndexer \
-${jobJarPath}
diff --git a/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh b/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh
deleted file mode 100644
index 87a62fca01..0000000000
--- a/druid/adhoc-jobs/src/scripts/RedisUserDataIndexer.sh
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-home=`echo $HOME`
-jobJarPath="${home}/adhoc-jobs-1.0/adhoc-jobs-1.0.jar"
-jobConfPath="${home}/adhoc-jobs-1.0/resources/cassandraRedis.conf"
-
-spark/bin/spark-submit \
---conf spark.driver.extraJavaOptions="-Dconfig.file=${jobConfPath}" \
---class org.ekstep.analytics.jobs.CassandraRedisIndexer \
-${jobJarPath}
diff --git a/druid/adhoc-jobs/src/test/resources/application.conf b/druid/adhoc-jobs/src/test/resources/application.conf
deleted file mode 100644
index fcdc438309..0000000000
--- a/druid/adhoc-jobs/src/test/resources/application.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-elasticsearch.host="localhost"
-elasticsearch.port="9200"
-elasticsearch.scroll.size="1000"
-elasticsearch.query.index="compositesearch"
-elasticsearch.query.jsonString="{\"query\":{\"match_all\":{}}}"
-outputFilePath=""
-cloudStorage.container="telemetry-data-store"
-cloudStorage.objectKey="druid-content-snapshot/snapshot.txt"
-cloudStorage.provider="azure"
-cloudStorage.accountName=""
-cloudStorage.accountKey=""
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala
deleted file mode 100644
index 679fcc6064..0000000000
--- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestCassandraRedisIndexer.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-
-class TestCassandraRedisIndexer extends FlatSpec with Matchers with BeforeAndAfterAll {
-
-    "cassandra indexer" should  "index user data to redis" in {
-        println(s"======> Starting Redis indexing job <=======")
-        //CassandraToRedisIndexer.main(Array())
-    }
-}
\ No newline at end of file
diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala
deleted file mode 100644
index ef443de3ee..0000000000
--- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESCloudUploader.scala
+++ /dev/null
@@ -1,9 +0,0 @@
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-
-class TestESCloudUploader extends FlatSpec with Matchers with BeforeAndAfterAll {
-
-    "main method" should  "run" in {
-        println(s"======> Starting Redis indexing job <=======")
-        //ESToCloudUploader.main(Array())
-    }
-}
diff --git a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala b/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala
deleted file mode 100644
index 58c07dbccf..0000000000
--- a/druid/adhoc-jobs/src/test/scala/org/ekstep/analytics/jobs/TestESRedisIndexer.scala
+++ /dev/null
@@ -1,11 +0,0 @@
-package org.ekstep.analytics.jobs
-
-import org.scalatest.{BeforeAndAfterAll, FlatSpec, Matchers}
-
-class TestESRedisIndexer extends FlatSpec with Matchers with BeforeAndAfterAll {
-
-    "main method" should  "run" in {
-        println(s"======> Starting Redis indexing job <=======")
-        //ESToRedisIndexer.main(Array())
-    }
-}
diff --git a/druid/ingestion-spec/content_index_batch.json b/druid/ingestion-spec/content_index_batch.json
index 9a29c0b26c..3167999662 100644
--- a/druid/ingestion-spec/content_index_batch.json
+++ b/druid/ingestion-spec/content_index_batch.json
@@ -776,14 +776,9 @@
     "ioConfig":{
       "type":"index",
       "firehose" : {
-        "type" : "static-azure-blobstore",
-        "blobs": [
-          {
-            "container": "telemetry-data-store",
-            "path": "/druid-content-snapshot/snapshot.txt"
-          }
-        ],
-        "fetchTimeout": 120000
+        "type" : "static-s3",
+        "uris": [ "s3://odev-dev-diksha-telemetry/druid-content-snapshot/snapshot.txt"],
+        "fetchTimeout": 300000
       }
     },
     "tuningConfig":{
@@ -794,4 +789,4 @@
       "logParseExceptions": true
     }
   }
-}
\ No newline at end of file
+}
diff --git a/druid/ingestion-spec/rollups/ml_project_index_rollup.json b/druid/ingestion-spec/rollups/ml_project_index_rollup.json
new file mode 100644
index 0000000000..57f23ec9e4
--- /dev/null
+++ b/druid/ingestion-spec/rollups/ml_project_index_rollup.json
@@ -0,0 +1,258 @@
+{
+    "type": "kafka",
+    "spec": {
+      "dataSchema": {
+        "dataSource": "ml-project-rollup",
+        "parserSpec": {
+            "format": "json",
+            "flattenSpec": {
+                "useFieldDiscovery": true,
+                "fields": [
+                  {
+                    "type": "path",
+                    "name": "project_title",
+                    "expr": "$.solutionInformation.name"
+                  },
+                  {
+                    "type": "path",
+                    "name": "area_of_improvement",
+                    "expr": "$.categories[*].name"
+                  },
+                  {
+                    "type": "path",
+                    "name": "tasks",
+                    "expr": "$.tasks[*].name"
+                  },
+                  {
+                    "type": "path",
+                    "name": "tasks_status",
+                    "expr": "$.tasks[*].status"
+                  },
+                  {
+                    "type": "path",
+                    "name": "designation",
+                    "expr": "$.userRoleInformation.role"
+                  },
+                  {
+                    "type": "path",
+                    "name": "task_evidence",
+                    "expr": "$.tasks[*].attachments[*].sourcePath"
+                  },
+                  {
+                    "type": "path",
+                    "name": "task_id",
+                    "expr": "$.tasks[*]._id"
+                  },
+                  {
+                    "type": "path",
+                    "name": "program_id",
+                    "expr": "$.programId"
+                  },
+                  {
+                    "type": "path",
+                    "name": "status_of_project",
+                    "expr": "$.status"
+                  },
+                  {
+                    "type": "path",
+                    "name": "program_name",
+                    "expr": "$.programInformation.name"
+                  },
+                  {
+                    "type": "path",
+                    "name": "project_updated_date",
+                    "expr": "$.updatedAt"
+                  },
+                  {
+                    "type": "path",
+                    "name": "createdBy",
+                    "expr": "$.userid"
+                  },
+                  {
+                    "type": "path",
+                    "name": "program_externalId",
+                    "expr": "$.programExternalId"
+                  },
+                  {
+                    "type": "path",
+                    "name": "private_program",
+                    "expr": "$.isAPrivateProgram"
+                  },
+                  {
+                    "type": "path",
+                    "name": "project_terms_and_condition",
+                    "expr": "$.hasAcceptedTAndC"
+                  },
+                  {
+                    "type": "path",
+                    "name": "solution_id",
+                    "expr": "$.solutionInformation._id"
+                  },
+                  {
+                    "type": "path",
+                    "name": "state_externalId",
+                    "expr": "$.userRoleInformation.state"
+                  },
+                  {
+                    "type": "path",
+                    "name": "block_externalId",
+                    "expr": "$.userRoleInformation.block"
+                  },
+                  {
+                    "type": "path",
+                    "name": "district_externalId",
+                    "expr": "$.userRoleInformation.district"
+                  },
+                  {
+                    "type": "path",
+                    "name": "cluster_externalId",
+                    "expr": "$.userRoleInformation.cluster"
+                  },
+                  {
+                    "type": "path",
+                    "name": "school_externalId",
+                    "expr": "$.userRoleInformation.school"
+                  }
+                ]
+              }
+        },
+        "timestampSpec": {
+          "column": "project_updated_date",
+          "format": "iso",
+          "missingValue": null
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "project_title"
+            },
+            {
+              "type": "string",
+              "name": "area_of_improvement"
+            },
+            {
+              "type": "string",
+              "name": "tasks"
+            },
+            {
+              "type": "string",
+              "name": "tasks_status"
+            },
+            {
+              "type": "string",
+              "name": "designation"
+            },
+            {
+              "type": "string",
+              "name": "task_evidence"
+            },
+            {
+              "type": "string",
+              "name": "task_id"
+            },
+            {
+              "type": "string",
+              "name": "status_of_project"
+            },
+            {
+              "type": "string",
+              "name": "program_name"
+            },
+            {
+              "type": "string",
+              "name": "createdBy"
+            },
+            {
+              "type": "string",
+              "name": "program_externalId"
+            },
+            {
+              "type": "string",
+              "name": "private_program"
+            },
+            {
+              "type": "string",
+              "name": "project_terms_and_condition"
+            },
+            {
+              "type": "string",
+              "name": "state_externalId"
+            },
+            {
+              "type": "string",
+              "name": "block_externalId"
+            },
+            {
+              "type": "string",
+              "name": "district_externalId"
+            },
+            {
+              "type": "string",
+              "name": "cluster_externalId"
+            }
+          ],
+          "dimensionExclusions": []
+        },
+        "metricsSpec": [
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_unique_programs",
+            "fieldName": "program_id"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_createdBy",
+            "fieldName": "createdBy"
+          },
+          {
+            "type": "HLLSketchBuild",
+            "name": "count_of_solution_id",
+            "fieldName": "solution_id"
+          }
+        ],
+        "granularitySpec": {
+          "type": "uniform",
+          "segmentGranularity": "DAY",
+          "queryGranularity": "HOUR",
+          "rollup": true
+        },
+        "transformSpec": {
+          "filter": null,
+          "transforms": [
+            {
+              "type": "expression",
+              "name": "task_evidence_status",
+              "expression": "if(array_length(\"task_evidence\") > 0, 'true', 'false')"
+            },
+            {
+              "type": "expression",
+              "name": "parent_channel",
+              "expression": "upper('SHIKSHALOKAM')"
+            },
+            {
+              "type": "expression",
+              "name": "project_created_type",
+              "expression": "if((NULL != \"project_created_type\"), 'project imported from library', 'user created project')"
+            }
+          ]
+        }
+      },
+      "ioConfig": {
+        "topic": "{{env}}.ml.project",
+        "replicas": 1,
+        "taskCount": "{{rollup_ml_project_taskcount}}",
+        "taskDuration": "PT14400S",
+        "consumerProperties": {
+          "bootstrap.servers": "{{kafka_brokers}}"
+        },
+        "useEarliestOffset": true,
+        "completionTimeout": "PT1800S"
+      },
+      "tuningConfig": {
+        "type": "kafka",
+        "reportParseExceptions": false,
+        "maxRowsPerSegment": 5000000
+      }
+    }
+  }
diff --git a/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json b/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json
new file mode 100644
index 0000000000..6ab6767584
--- /dev/null
+++ b/druid/ingestion-spec/rollups/tpd_index_hourly_syncts.json
@@ -0,0 +1,168 @@
+{
+  "type": "kafka",
+  "dataSchema": {
+    "dataSource": "tpd-hourly-rollup-syncts",
+    "parser": {
+      "type": "string",
+      "parseSpec": {
+        "format": "json",
+        "flattenSpec": {
+          "useFieldDiscovery": false,
+          "fields": [
+            {
+              "type": "root",
+              "name": "eid"
+            },
+            {
+              "type": "path",
+              "name": "collection_type",
+              "expr": "$.collectiondata.contenttype"
+            },
+            {
+              "type": "path",
+              "name": "context_env",
+              "expr": "$.context.env"
+            },
+            {
+              "type": "path",
+              "name": "edata_id",
+              "expr": "$.edata.id"
+            },
+            {
+              "type": "path",
+              "name": "edata_mode",
+              "expr": "$.edata.mode"
+            },
+            {
+              "type": "path",
+              "name": "edata_pageid",
+              "expr": "$.edata.pageid"
+            },
+            {
+              "type": "path",
+              "name": "edata_type",
+              "expr": "$.edata.type"
+            },
+            {
+              "type": "path",
+              "name": "user_signin_type",
+              "expr": "$.userdata.usersignintype"
+            },
+            {
+              "type": "path",
+              "name": "actor_id",
+              "expr": "$.actor.id"
+            },
+            {
+              "type": "path",
+              "name": "context_did",
+              "expr": "$.context.did"
+            }
+          ]
+        },
+        "dimensionsSpec": {
+          "dimensions": [
+            {
+              "type": "string",
+              "name": "eid"
+            },
+            {
+              "type": "string",
+              "name": "context_env"
+            },
+            {
+              "type": "string",
+              "name": "user_signin_type"
+            },
+            {
+              "type": "string",
+              "name": "collection_type"
+            },
+            {
+              "type": "string",
+              "name": "edata_id"
+            },
+            {
+              "type": "string",
+              "name": "edata_mode"
+            },
+            {
+              "type": "string",
+              "name": "edata_pageid"
+            },
+            {
+              "type": "string",
+              "name": "edata_type"
+            },
+            {
+              "type":"string",
+              "name": "first_time_user"
+            }
+          ]
+        },
+        "timestampSpec": {
+          "column": "syncts",
+          "format": "auto"
+        }
+      }
+    },
+    "transformSpec": {
+      "transforms": [
+        {
+          "type": "expression",
+          "name": "first_time_user  ",
+          "expression": "if(like(\"context_cdata_type\",'FirstTimeUser') && like(\"context_cdata_id\",'true') , 'true','false')"
+        },
+        {
+          "type": "expression",
+          "name": "is_playerstart_event",
+          "expression": "if((like(\"context_pdata_pid\",'%contentplayer%'), if ( \"eid\" == 'START', 'true','false') ,'true')"
+        }
+      ],
+      "filter": {
+        "type": "selector",
+        "dimension": "is_playerstart_event",
+        "value": "true"
+      }
+    },
+    "metricsSpec": [
+      {
+        "name": "total_count",
+        "type": "count"
+      },
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_devices",
+        "fieldName": "context_did"
+      },
+      {
+        "type": "HLLSketchBuild",
+        "name": "unique_users",
+        "fieldName": "actor_id"
+      }
+    ],
+    "granularitySpec": {
+      "type": "uniform",
+      "segmentGranularity": "hour",
+      "queryGranularity": "hour",
+      "rollup": true
+    }
+  },
+  "ioConfig": {
+    "topic": "{{env}}.druid.events.telemetry",
+    "consumerProperties": {
+      "bootstrap.servers": "{{kafka_brokers}}"
+    },
+    "taskCount": 1,
+    "replicas": 1,
+    "taskDuration": "PT7200S",
+    "completionTimeout" : "PT3600S",
+    "useEarliestOffset": true
+  },
+  "tuningConfig": {
+    "type": "kafka",
+    "reportParseExceptions": false,
+    "logParseExceptions": true,
+    "maxSavedParseExceptions": 10
+  }
+}
diff --git a/kubernetes/ansible/bootstrap_minimal.yaml b/kubernetes/ansible/bootstrap_minimal.yaml
new file mode 100644
index 0000000000..f9facd6152
--- /dev/null
+++ b/kubernetes/ansible/bootstrap_minimal.yaml
@@ -0,0 +1,44 @@
+---
+# tasks file for bootstrap-k8s
+
+- hosts: local
+  gather_facts: false
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  serial: 1
+  vars:
+    helm_vars:
+      reloader:
+        watchGlobally: false
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  pre_tasks:
+    - name: Creating tmp file
+      tempfile:
+        state: file
+        suffix: helm
+      register: helm_file
+    - name: Creating helm_charts vars file
+      copy:
+        content: "{{ helm_vars|to_yaml }}"
+        dest: "{{ helm_file.path }}"
+    - name: Creating namespace
+      shell: "kubectl create namespace {{ item }} "
+      with_items:
+        - "flink-{{ env }}"
+        - "flink-kp-{{ env }}"
+        - secor
+        - monitoring
+        - logging
+      ignore_errors: yes
+
+    - name: Creating docker secrets
+      shell: "kubectl create secret docker-registry {{ imagepullsecrets }} --namespace {{ item }} --docker-server {{ core_vault_docker_registry_url }} --docker-username {{ core_vault_docker_registry_user }} --docker-password {{ core_vault_docker_registry_password }} --dry-run=client -o=yaml | kubectl apply -f -"
+      with_items:
+        - "flink-{{ env }}"
+        - "flink-kp-{{ env }}"
+        - secor
+      when: imagepullsecrets|length > 0
+
+    - name: Installing reloader for configmaps reload
+      shell: helm upgrade --install --atomic reloader ../helm_charts/bootstrap/reloader --namespace secor -f "{{ helm_file.path }}"
diff --git a/kubernetes/ansible/druid_provision.yml b/kubernetes/ansible/druid_provision.yml
new file mode 100644
index 0000000000..a3635f03b5
--- /dev/null
+++ b/kubernetes/ansible/druid_provision.yml
@@ -0,0 +1,11 @@
+---
+- hosts: local
+  gather_facts: no
+  vars:
+    - cluster_type: "{{ cluster_type }}"
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  roles:
+    - druid
\ No newline at end of file
diff --git a/kubernetes/ansible/flink_jobs_stop_start.yml b/kubernetes/ansible/flink_jobs_stop_start.yml
new file mode 100644
index 0000000000..639ca1529c
--- /dev/null
+++ b/kubernetes/ansible/flink_jobs_stop_start.yml
@@ -0,0 +1,43 @@
+---
+- hosts: local
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  tasks:
+    - name: scale down the taskmanager to zero
+      shell: kubectl scale deployment {{ item }}-taskmanager --replicas=0 -n {{ flink_namespace }}
+      with_items:
+        - "{{ job_names.split(',')|list }}"
+      tags:
+        - scaledown_taskmanager
+ 
+    - name: scale down the jobmanager
+      shell: kubectl get job {{ item }}-jobmanager -o json -n {{ flink_namespace }} | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels."controller-uid")' |jq '.spec.parallelism = 0' | kubectl replace --force -f -
+      with_items:
+        - "{{ job_names.split(',')|list }}"
+      tags:
+        - scaledown_jobmanager
+  
+    - name: scale up the jobmanager
+      shell: kubectl get job {{ item }}-jobmanager -o json -n {{ flink_namespace }} | jq 'del(.spec.selector)' | jq 'del(.spec.template.metadata.labels."controller-uid")' |jq '.spec.parallelism = 1' | kubectl replace --force -f -
+      with_items:
+        - "{{ job_names.split(',')|list }}"
+      tags:
+        - scaleup_jobmanager
+  
+    - name: Scale up taskmanager
+      shell: kubectl scale deployment {{ item }}-taskmanager --replicas={{ flink_job_names['%s'|format(item)].replica }} -n {{ flink_namespace }}
+      with_items:
+        - "{{ job_names.split(',')|list }}"
+      tags:
+        - scaleup_taskmanager
+
+    - name: rollout the taskmanager
+      shell: kubectl rollout restart deployment {{ item }}-taskmanager -n {{ flink_namespace }}
+      with_items:
+        - "{{ job_names.split(',')|list }}"
+      tags:
+        - scaleup_taskmanager
+
diff --git a/kubernetes/ansible/kafka_provision.yaml b/kubernetes/ansible/kafka_provision.yaml
new file mode 100644
index 0000000000..7e2a3ede1e
--- /dev/null
+++ b/kubernetes/ansible/kafka_provision.yaml
@@ -0,0 +1,10 @@
+---
+- hosts: local
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  roles:
+    - kafka
+  tags: kafka
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/druid/defaults/main.yml b/kubernetes/ansible/roles/druid/defaults/main.yml
new file mode 100644
index 0000000000..509b3d6dcf
--- /dev/null
+++ b/kubernetes/ansible/roles/druid/defaults/main.yml
@@ -0,0 +1,269 @@
+druid_image: "apache/druid:0.21.1"
+druid_namespace: "druid-{{ cluster_type }}"
+
+# Druid Template variables
+druid_monitoring: false
+
+# postgres:
+#   db_url: "{{ groups['postgres'][0] }}"
+#   db_port: 5432
+#   db_admin_user: analytics
+#   db_admin_password: "{{dp_vault_pgdb_admin_password}}"
+#   druid:
+#     db_name: "druid-{{ cluster_type }}"
+#     db_username: "druid"
+#     db_password: "{{dp_vault_druid_postgress_pass}}"
+
+# druid_user: druid
+druid_directory: "/opt/druid"
+
+###################### Common Varibles ###########################
+
+druid_default_tmp_dir: "/var/tmp"
+druid_log_dir: "/var/log/druid/"
+druid_gc_logdir: "/var/log/druid/crash_logs"
+druid_crash_logdir: "/var/log/druid/crash_logs"
+
+
+#Writing request query logs to file 
+# druid_request_logging_type: "file"
+druid_request_logging_type: "composing"
+
+#Druid Extensions
+
+druid_extensions_loadList : '"druid-azure-extensions", "postgresql-metadata-storage", "druid-kafka-indexing-service"'
+druid_community_extensions:
+  - "graphite-emitter"
+druid_emitter_logging_logLevel: INFO
+
+# End of druid_extensions
+
+druid_indexing_logs_type: "azure"
+druid_indexing_log_dir: "/var/druid/indexing-logs"
+druid_indexing_storage_type: "metadata"
+druid_indexing_task_basedir: "/var/task"
+
+druid_common_monitors: '"com.metamx.metrics.JvmMonitor","org.apache.druid.java.util.metrics.JvmMonitor"'
+# druid_common_emitters: '"logging","graphite"'
+druid_common_emitters: '"logging"'
+
+druid_graphite_prefix: "druid.metric"
+druid_whitelist_filepath: "{{ druid_directory }}/whitelist"
+
+#End of Common variables
+
+######################## Coordinator Varibles #########################
+
+druid_coordinator_port: 8081
+druid_coordinator_service : druid/coordinator
+
+druid_coordinator_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_coordinator_gc_logfile: "{{ druid_crash_logdir }}/gc.log"
+druid_coordinator_heap_dump_file : "{{ druid_crash_logdir }}/coordinator.hprof"
+
+####################### Overlord Varibles #############################
+
+druid_overlord_port: 8090
+druid_overlord_service : druid/overlord
+
+druid_overlord_heap_size: 256m
+druid_overlord_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_overlord_gc_logfile: "{{ druid_crash_logdir }}/gc_overlord.log"
+druid_overlord_heap_dump_file : "{{ druid_crash_logdir }}/overlord.hprof"
+
+######################### Broker Varibles #############################
+
+druid_broker_port: 8082
+druid_broker_service: druid/broker
+
+druid_broker_max_direct_size: 800m
+druid_broker_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_broker_gc_logdir: "{{ druid_crash_logdir }}/gc.log"
+druid_broker_heap_dump_file : "{{ druid_crash_logdir }}/broker.hprof"
+
+####################### Historical Varibles ###########################
+
+druid_historical_port: 8084
+druid_historical_service: druid/historical
+
+druid_historical_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_historical_gc_logfile: "{{ druid_crash_logdir }}/historical.gc.log"
+druid_historical_heap_dump_file: "{{ druid_crash_logdir }}/historical.hprof"
+
+druid_historical_monitoring_monitors: '"org.apache.druid.server.metrics.HistoricalMetricsMonitor","com.metamx.metrics.JvmMonitor"'
+
+# druid_historical_segmentcache_path: "/var/segments/store"
+druid_historical_cache_size: 256MiB
+druid_historical_cache_expiry: 3600000
+
+druid_historical_cache_unCacheable: '"select", "scan"'
+
+##################### Middlemanager Varibles ##########################
+
+druid_middlemanager_port: 8091
+druid_middlemanager_service: druid/middlemanager
+
+druid_middlemanager_tmp_dir: "{{ druid_default_tmp_dir }}"
+druid_middlemanager_gc_logdir: "{{ druid_crash_logdir }}/gc.log"
+
+druid_middlemanager_peon_segmentcache_path: "/var/segment_cache"
+druid_azure_container_name: "telemetry-data-store"
+druid_mm_heap_dump_file : "{{ druid_crash_logdir }}/middlemanager.hprof"
+
+##################### Middlemanager Varibles ##########################
+
+druid_router_service: druid/router
+druid_router_plaintextPort: 8888
+
+enable_druid_sql: true
+persistent_storage_class: managed-premium
+
+druid_configs:
+  raw:
+   #Druid Postgres Details
+   druid_postgres_db: "druid-{{ cluster_type }}"
+   druid_postgres_host: "{{ postgres.db_url }}"
+   druid_postgres_port: "{{ postgres.db_port }}"
+   druid_postgres_user: "druid@{{ postgres.db_url }}"
+   druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+   #Druid Azure Details
+   druid_storage_type: "azure"
+   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
+   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
+   azure_container: "{{ druid_azure_container_name }}"
+   #Logging the indexing logs to azure
+   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_folder: "druidlogs"
+   druid_coordinator_heap_size: 128M
+   druid_coordinator_period: PT30S
+   druid_coordinator_startDelay: PT30S
+   druid_coordinator_balance_strategy: diskNormalized
+   druid_coordinator_monitoring_monitor_enabled: false
+   druid_coordinator_replicas: 1
+   #Druid overlord node configuration
+   druid_overlord_heap_size: 256M
+   druid_overlord_monitoring_monitor_enabled: false
+   druid_overlord_replicas: 1
+   #Druid broker node configuration
+   druid_broker_min_heap_size: 128M
+   druid_broker_max_heap_size: 128M
+   druid_broker_max_direct_size: 800M
+   druid_broker_http_numConnections: 5
+   druid_broker_server_http_numThread: 25
+   druid_broker_processing_bufferBytes: 128MiB
+   druid_broker_processing_threads: 2
+   druid_broker_monitor_enabled: false
+   druid_broker_replicas: 1
+   #Druid historical node configuration
+   #druid_historical_min_heap_size: 1048m
+   #druid_historical_max_heap_size: 1048m
+   druid_historical_min_heap_size: 512M
+   druid_historical_max_heap_size: 512M
+   druid_historical_max_direct_size: 800M
+   druid_historical_persistent_volume_size: 20G
+   druid_historical_http_numConnections: 5
+   druid_historical_server_http_numThread: 25
+   druid_historical_processing_bufferBytes: 128MiB
+   druid_historical_processing_threads: 2
+   druid_historical_enable_cache: false
+   #druid_historical_segmentcache_size: 10000000000
+   druid_historical_segmentcache_size: 2000000000
+   druid_historical_server_maxsize: 1000000000
+   druid_historical_processing_num_merge_buffers: 2
+   druid_query_ondiskstorage_enabled: true
+   druid_historical_maxMergingDictionarySize: 100000000
+   druid_historical_segmentcache_numloadingthreads: 4
+   druid_historical_segmentcache_path: "/var/segments/store"
+   druid_query_groupBy_maxOnDiskStorage: 1073741824
+   druid_historical_monitoring_monitor_enabled: false
+   druid_historical_replicas: 1
+   #Druid middlemanager configuration
+   druid_middlemanager_heap_size: 128M
+   druid_middlemanager_worker_cap: 2
+   druid_middlemanager_peon_heap_size: 256M
+   druid_middlemanager_persistent_volume_size: 20G
+   druid_mm_java_opts_array: "-server -Xms512m -Xmx512m -XX:+UseG1GC -XX:MaxGCPauseMillis=100"
+   druid_middlemanager_peon_server_http_numThread: 25
+   druid_middlemanager_peon_processing_bufferBytes: 25MiB
+   druid_middlemanager_peon_processing_threads: 2
+   druid_middlemanager_peon_server_maxsize: 0
+   druid_middlemanager_monitoring_monitor_enabled: false
+   druid_indexing_queue_startDelay: PT30S
+   druid_middlemanager_replicas: 1
+   #Druid Router configurations
+   druid_router_heap_size: 512M
+   druid_router_http_numConnections: 50
+   druid_router_http_readTimeout: PT5M
+   druid_router_http_numMaxThreads: 100
+   druid_server_http_numThreads: 100
+   druid_router_managementProxy_enabled: true
+   druid_router_replicas: 1
+  rollup:
+   #Druid Postgres Details
+   druid_postgres_db: "druid-{{ cluster_type }}"
+   druid_postgres_host: "{{ postgres.db_url }}"
+   druid_postgres_port: "{{ postgres.db_port }}"
+   druid_postgres_user: "druid@{{ postgres.db_url }}"
+   #Druid Azure Details
+   druid_postgres_pass: "{{ dp_vault_druid_postgress_pass }}"
+   azure_account_name: "{{ sunbird_druid_storage_account_name }}"
+   azure_storage_secret: "{{ sunbird_druid_storage_account_key }}"
+   azure_container: "{{ druid_azure_container_name }}"
+   #Logging the indexing logs to azure
+   druid_log_azure_container: "{{ druid_azure_container_name }}"
+   druid_log_azure_folder: "druidlogs"
+   druid_coordinator_heap_size: 128m
+   druid_coordinator_period: PT30S
+   druid_coordinator_startDelay: PT30S
+   druid_coordinator_balance_strategy: diskNormalized
+   druid_coordinator_monitoring_monitor_enabled: false
+   druid_coordinator_replicas: 1
+   #Druid overlord node configuration
+   druid_overlord_heap_size: 256M
+   druid_overlord_monitoring_monitor_enabled: false
+   druid_overlord_replicas: 1
+   #Druid broker node configuration
+   druid_broker_min_heap_size: 128m
+   druid_broker_max_heap_size: 128m
+   druid_broker_max_direct_size: 700m
+   druid_broker_http_numConnections: 5
+   druid_broker_server_http_numThread: 25
+   druid_broker_processing_bufferBytes: 134217728
+   druid_broker_processing_threads: 2
+   druid_broker_monitor_enabled: false
+   druid_broker_replicas: 1
+   #Druid historical node configuration
+   druid_historical_min_heap_size: 1048m
+   druid_historical_max_heap_size: 1048m
+   druid_historical_max_direct_size: 800m
+   druid_historical_http_numConnections: 5
+   druid_historical_server_http_numThread: 25
+   druid_historical_processing_bufferBytes: 134217728
+   druid_historical_processing_threads: 2
+   druid_historical_enable_cache: false
+   druid_historical_segmentcache_size: 2000000000
+   druid_historical_server_maxsize: 10000000000
+   druid_historical_processing_num_merge_buffers: 2
+   druid_query_ondiskstorage_enabled: false
+   druid_historical_segmentcache_numloadingthreads: 4
+   druid_historical_segmentcache_path: "/var/segments/store"
+   druid_historical_replicas: 1
+   #Druid middlemanager configuration
+   druid_middlemanager_heap_size: 128m
+   druid_middlemanager_worker_cap: 4
+   druid_middlemanager_peon_heap_size: 512m
+   druid_mm_java_opts_array: "-server -Xms256m -Xmx256m -XX:+UseG1GC -XX:MaxGCPauseMillis=100"
+   druid_middlemanager_peon_server_http_numThread: 25
+   druid_middlemanager_peon_processing_bufferBytes: 25000000
+   druid_middlemanager_peon_processing_threads: 2
+   druid_middlemanager_peon_server_maxsize: 0
+   druid_indexing_queue_startDelay: PT30S
+   druid_middlemanager_replicas: 1
+   #Druid router configuration
+   druid_router_heap_size: 1g
+   druid_router_http_numConnections: 50
+   druid_router_http_readTimeout: PT5M
+   druid_router_http_numMaxThreads: 100
+   druid_server_http_numThreads: 100
+   druid_router_managementProxy_enabled: true
+   druid_router_replicas: 1
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/druid/tasks/main.yml b/kubernetes/ansible/roles/druid/tasks/main.yml
new file mode 100644
index 0000000000..2ff7190516
--- /dev/null
+++ b/kubernetes/ansible/roles/druid/tasks/main.yml
@@ -0,0 +1,37 @@
+- name: template values.yaml file
+  template:
+    src: "{{ chart_base_path }}/druid-cluster/values.j2"
+    dest: "{{ chart_base_path }}/druid-cluster/values.yaml"
+
+- name: Create druid database if not exists
+  postgresql_db: name="{{ postgres.druid.db_name }}" \
+            login_host="{{ postgres.db_url }}" \
+            port="{{ postgres.db_port }}" \
+            login_user="{{ postgres.db_admin_user }}" \
+            login_password="{{ postgres.db_admin_password }}" \
+            encoding='UTF-8' \
+            state=present
+
+- name: Create druid user if not exists
+  postgresql_user: name="druid" \
+              password="{{ dp_vault_druid_postgress_pass }}" \
+              no_password_changes=true \
+              priv=ALL \
+              state=present \
+              login_host="{{ postgres.db_url }}" \
+              port="{{ postgres.db_port }}" \
+              login_user="{{ postgres.db_admin_user }}" \
+              login_password="{{ postgres.db_admin_password }}" \
+              db="{{ postgres.druid.db_name }}"
+
+- name: Install druid operator
+  shell: helm upgrade --install -n {{ druid_namespace }} --set env.WATCH_NAMESPACE="{{ druid_namespace }}" druid-operator {{ chart_base_path }}/druid-operator
+
+- name: Install nginx-controller
+  shell: helm upgrade --install -n {{ druid_namespace }} druid-{{ cluster_type }} {{ chart_base_path }}/nginx-ingress --skip-crds
+
+- name: Install dependencies
+  shell: helm dependency update {{ chart_base_path }}/druid-cluster
+
+- name: Install druid cluster
+  shell: helm upgrade --install -n {{ druid_namespace }} druid-cluster {{ chart_base_path }}/druid-cluster
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
index c2e52f22dd..633026b825 100644
--- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
+++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml
@@ -1,6 +1,6 @@
-flink_namespace: flink-dev
-imagepullsecrets: sunbird-registry-secret
-
+flink_namespace: "flink-{{ env }}"
+imagepullsecrets: "{{ env }}-registry-secret"
+service_monitor_enabled: true
 ### Job manager related vars
 jobmanager_rpc_port: 6123
 jobmanager_blob_port: 6124
@@ -33,10 +33,18 @@ postgres_max_connections: 2
 azure_account: "{{ sunbird_private_storage_account_name }}"
 azure_secret: "{{ sunbird_private_storage_account_key }}"
 flink_container_name: dev-data-store
+flink_dp_storage_container: ""
+checkpoint_store_type: azure
 checkpoint_interval: 60000
+checkpoint_pause_between_seconds: 5000
+checkpoint_compression_enabled: true
 restart_attempts: 3
 restart_delay: 30000 # in milli-seconds
 producer_max_request_size: 1572864
+producer_batch_size: 98304
+producer_linger_ms: 10
+producer_compression: snappy
+
 redis_timeout: 30000
 device_profile_table: "{{ env }}_device_profile"
 
@@ -44,51 +52,70 @@ device_profile_table: "{{ env }}_device_profile"
 extractor_event_max_size: 1048576 # Max is only 1MB
 extractor_max_request_size: 5242880
 extractor_consumer_parallelism: 1
-dedup_parallelism: 1
-extraction_parallelism: 1
-redactor_parallelism: 1
+extractor_operators_parallelism: 1
 telemetry_extractor_key_expiry_seconds: 3600
 
+### Ingest Raw router job related vars
+ingest_router_consumer_parallelism: 1
+ingest_router_operators_parallelism: 1
+raw_router_consumer_parallelism: 1
+raw_router_downstream_parallelism: 1
+
 ### Pipeline-preprocessor related vars
 pipeline_preprocessor_consumer_parallelism: 1
-telemetry_validation_parallelism: 1
-telemetry_router_parallelism: 1
-share_events_flattener_parallelism: 1
-portal_id: dev.sunbird.portal
-desktop_id: dev.sunbird.desktop
+pipeline_preprocessor_operators_parallelism: 1
+portal_id: "{{ env }}.sunbird.portal"
+desktop_id: "{{ env }}.sunbird.desktop"
 pipeline_preprocessor_key_expiry_seconds: 3600
 
 ### De-normalization related vars
 denorm_consumer_parallelism: 1
-device_denorm_parallelism: 1
-user_denorm_parallelism: 1
-content_denorm_parallelism: 1
-loc_denorm_parallelism: 1
-dialcode_denorm_parallelism: 1
-denorm_parallelism: 1
-denorm_sink_parallelism: 1
-denorm_summary_dedup_parallelism: 1
-summary_sink_parallelism: 1
+telemetry_denorm_operators_parallelism: 1
 de_normalization_duplicationstore_key_expiry_seconds: 3600
 de_normalization_key_expiry_seconds: 3600
+denorm_window_count: 30
+denorm_window_shards: 1400
+
+denorm_secondary_consumer_parallelism: 1
+telemetry_denorm_secondary_operators_parallelism: 1
+denorm_primary_consumer_parallelism: 1
+telemetry_denorm_primary_operators_parallelism: 1
+
+redis_meta_content_port: "{{ content_port }}"
+redis_meta_device_port: "{{ device_port }}"
+redis_meta_user_port: "{{ user_port }}"
+redis_meta_dialcode_port: "{{ dialcode_port }}"
+
+redis_meta_content_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}"
+redis_meta_device_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}"
+redis_meta_user_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}"
+redis_meta_dialcode_host: "{% if metadata2_redis_host is defined %}{{ metadata2_redis_host }}{% else %}{{ redis_host }}{% endif %}"
+
+denorm_secondary_window_count: 30
+denorm_secondary_window_shards: 1400
+denorm_primary_window_count: 30
+denorm_primary_window_shards: 1400
+
+denorm_summary_window_count: 5
+denorm_summary_window_shards: 1400
 
 ### summary-denormalization related vars
 summary_denorm_consumer_parallelism: 1
-summary_denorm_parallelism: 1
-summary_denorm_sink_parallelism: 1
-summary_denorm_dedup_parallelism: 1
-summary_denorm_summary_sink_parallelism: 1
+summary_denorm_operators_parallelism: 1
 summary_denorm_duplication_key_expiry_seconds: 3600
 summary_denorm_key_expiry_seconds: 3600
 
 ### Druid-validator related vars
 druid_validator_consumer_parallelism: 1
-validator_parallelism: 1
-router_parallelism: 1
+druid_validator_operators_parallelism: 1
 druid_validator_key_expiry_seconds: 3600
 druid_validation_enabled: true
 druid_deduplication_enabled: true
 
+### error-denormalization related vars
+error_denorm_consumer_parallelism: 1
+error_denorm_operators_parallelism: 1
+
 ### Device-profile-updater related vars
 deviceprofile_parallelism: 1
 device_profile_updater_key_expiry_seconds: 3600
@@ -96,7 +123,7 @@ device_profile_updater_key_expiry_seconds: 3600
 ### content-cache-updater
 dialcode_host: "{{ proto }}://{{ domain_name }}"
 dialcode_api_url: "{{ dialcode_host }}/{{ dialcode_endpoint }}"
-dialcode_api_auth_key: "{{ dp_vault_dialcode_api_auth_key }}"
+dialcode_api_auth_key: "{{ sunbird_api_auth_token }}"
 
 ### User-cache-updater related vars
 usercache_updater_parallelism: 1
@@ -105,11 +132,23 @@ middleware_cassandra_keyspace: sunbird
 middleware_cassandra_user_table: user
 middleware_cassandra_location_table: location
 
+#(user read api details)
+user_read_api_endpoint: "/learner/private/user/v1/read/"
+user_read_api_url : "http://{{private_ingressgateway_ip}}{{ user_read_api_endpoint }}"
+
+
 ### assessment-aggregator related vars
 assessaggregator_parallelism: 1
+assessaggregator_consumer_parallelism: 1
+assessaggregator_downstream_parallelism: 1
+assessaggregator_scoreaggregator_parallelism: 1
 middleware_cassandra_courses_keyspace: sunbird_courses
 middleware_cassandra_assessment_aggregator_table: assessment_aggregator
 middleware_cassandra_assessment_question_type : question
+middleware_cassandra_user_enrolments_table: user_enrolments
+middleware_cassandra_user_activity_agg_table: user_activity_agg
+content_read_api_host: "http://dev.sunbirded.org"
+content_read_api_endpoint: "/api/content/v1/read/" 
 
 ### taskmanager related vars
 healthcheck: true
@@ -132,28 +171,176 @@ job_classname: ""
 flink_job_names:
   telemetry-extractor:
     job_class_name: 'org.sunbird.dp.extractor.task.TelemetryExtractorStreamTask'
-    replica: 2
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 500000
+    min_replica: 1
+    max_replica: 2
   pipeline-preprocessor:
     job_class_name: 'org.sunbird.dp.preprocessor.task.PipelinePreprocessorStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 1000000
+    min_replica: 1
+    max_replica: 2
   de-normalization:
     job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
+  de-normalization-v2:
+    job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false 
+  de-normalization-secondary:
+    job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 1000000
+    min_replica: 1
+    max_replica: 2 
+  de-normalization-primary:
+    job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 1000000
+    min_replica: 1
+    max_replica: 2      
   druid-validator:
     job_class_name: 'org.sunbird.dp.validator.task.DruidValidatorStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 1000000
+    min_replica: 1
+    max_replica: 2
   assessment-aggregator:
     job_class_name: 'org.sunbird.dp.assessment.task.AssessmentAggregatorStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
   content-cache-updater:
     job_class_name: 'org.sunbird.dp.contentupdater.task.ContentCacheUpdaterStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
   user-cache-updater:
     job_class_name: 'org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
+  user-cache-updater-v2:
+    job_class_name: 'org.sunbird.dp.usercache.task.UserCacheUpdaterStreamTaskV2'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
   summary-denormalization:
     job_class_name: 'org.sunbird.dp.denorm.task.SummaryDenormalizationStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 100
+    min_replica: 1
+    max_replica: 2
   device-profile-updater:
     job_class_name: 'org.sunbird.dp.deviceprofile.task.DeviceProfileUpdaterStreamTask'
     replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false
+  ingest-router:
+    job_class_name: 'org.sunbird.dp.ingestrouter.task.IngestRouterStreamTask'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: true
+    scale_target_value: 500000
+    min_replica: 1
+    max_replica: 2
+  error-denormalization:
+    job_class_name: 'org.sunbird.dp.denorm.task.DenormalizationStreamTask'
+    replica: 1
+    jobmanager_memory: 1024m
+    taskmanager_memory: 1024m
+    taskmanager_process_memory: 1700m
+    jobmanager_process_memory: 1600m
+    taskslots: 1
+    cpu_requests: 0.3
+    scale_enabled: false 
+
+### controlling the flink jobs log level
+flink_jobs_console_log_level: INFO
+flink_libraries_log_level: ERROR
diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml
index 852620d12f..500376c4b7 100644
--- a/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml
+++ b/kubernetes/ansible/roles/flink-jobs-deploy/tasks/main.yml
@@ -10,6 +10,6 @@
   ignore_errors: yes
 
 - name: helm upgrade
-  shell: helm upgrade --install {{ item }} {{ chart_path }} --set job_classname={{ flink_job_names['%s'|format(item)].job_class_name }} --set taskmanager.replicas={{ flink_job_names['%s'|format(item)].replica }} -n {{ flink_namespace }}
+  shell: helm upgrade --install {{ item }} {{ chart_path }} --set job_classname={{ flink_job_names['%s'|format(item)].job_class_name }} --set taskmanager.replicas={{ flink_job_names['%s'|format(item)].replica }} --set taskmanager.cpu_requests={{ flink_job_names['%s'|format(item)].cpu_requests }} -n {{ flink_namespace }}
   with_items:
     - "{{ job_names_to_deploy.split(',')|list }}"
diff --git a/kubernetes/ansible/roles/kafka/defaults/main.yml b/kubernetes/ansible/roles/kafka/defaults/main.yml
new file mode 100644
index 0000000000..c50d063e39
--- /dev/null
+++ b/kubernetes/ansible/roles/kafka/defaults/main.yml
@@ -0,0 +1,19 @@
+kafka_namespace: "kafka"
+kafka_image_repository: "bitnami/kafka"
+kafka_image_tag: "2.8.1-debian-10-r31"
+
+kafka_delete_topic_enable: true
+
+kafka_replica_count: 3
+
+# Kubernetes Service type for external access. It can be NodePort or LoadBalancer
+service_type: "LoadBalancer"
+service_port: 9092
+
+# PV config
+kafka_persistence_size: 50Gi
+
+#Zookeeper configs
+zookeeper_enabled: true
+zookeeper_heapsize: 256
+zookeeper_replica_count: 3
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/kafka/tasks/main.yml b/kubernetes/ansible/roles/kafka/tasks/main.yml
new file mode 100644
index 0000000000..365116555b
--- /dev/null
+++ b/kubernetes/ansible/roles/kafka/tasks/main.yml
@@ -0,0 +1,10 @@
+- name: template values.yaml file
+  template:
+    src: "{{ chart_base_path }}/kafka/values.j2"
+    dest: "{{ chart_base_path }}/kafka/values.yaml"
+
+- name: Install dependencies
+  shell: helm dependency update {{ chart_base_path }}/kafka
+
+- name: Install kafka cluster
+  shell: helm upgrade --install -n {{ kafka_namespace }} kafka-cluster {{ chart_base_path }}/kafka --create-namespace
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml
new file mode 100644
index 0000000000..85d6f0f625
--- /dev/null
+++ b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml
@@ -0,0 +1,571 @@
+secor_namespace: "secor"
+image_repository: "secor"
+image_tag: "0.29"
+image_pullPolicy: IfNotPresent
+imagepullsecrets: "{{ env }}-registry-secret"
+secor_storage_class: managed-premium
+
+# this varible is to control the monitoring deployment. override this value by adding into
+# private repo common.yml to not to create the secor lag alert rules. The scenorio is , if adopters does not have
+# monitoring stack then the secor deployment job will fail. To avoid that we introduced this var.
+secor_alertrule_enabled: true
+# image:
+#   prefix: {}
+#   repository: "secor"
+#   tag: "secor-0.29_cf79409_7018"
+#   pullPolicy: IfNotPresent
+
+secor_jobs:
+  raw-telemetry-backup:
+    replicas: 1
+    consumer_group: "{{env_name}}.telemetry.raw.backup"
+    service_name: "raw_telemetry_backup"
+    base_path: "raw"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{env_name}}.telemetry.raw"
+    kafka_broker_host: "{{ groups['processing-cluster-kafka']|join(',') }}"
+    zookeeper_quorum: "{{ groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    message_channel_identifier: ""
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  failed-telemetry-backup:
+    replicas: 1
+    service_name: "failed_telemetry_backup"
+    consumer_group: "{{ env_name }}.telemetry.failed.backup"
+    base_path: "failed"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.failed"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  unique-telemetry-backup:
+    replicas: 1
+    service_name: "unique_telemetry_backup"
+    consumer_group: "{{ env_name }}.telemetry.unique.backup"
+    base_path: "unique/raw"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.unique"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  denorm-events-backup:
+    replicas: 1
+    service_name: "denorm_events_backup"
+    consumer_group: "{{ env_name }}.telemetry.denorm.backup"
+    base_path: "telemetry-denormalized"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.denorm"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "true"
+    partition_prefix_key: "eid"
+    partition_prefix_mapping: '{"ME_WORKFLOW_SUMMARY":"summary","DEFAULT":"raw"}'
+    output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  derived-denorm-events-backup:
+    replicas: 1
+    service_name: "derived_denorm_events_backup"
+    consumer_group: "{{ env_name }}.summary.backup"
+    base_path: "telemetry-denormalized/summary"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.druid.events.summary"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser" 
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  channel-telemetry-backup:
+    replicas: 1
+    service_name: "channel_telemetry_backup"
+    consumer_group: "{{ env_name }}.telemetry.channel.backup"
+    base_path: "data-exhaust/raw"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.denorm"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: "eid"
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
+    message_channel_identifier: "derivedlocationdata.state"
+    message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  channel-summary-backup:
+    replicas: 1
+    service_name: "channel_summary_backup"
+    consumer_group: "{{ env_name }}.summary.channel.backup"
+    base_path: "data-exhaust/summary"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.druid.events.summary"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{kafkaPartition}-{currentTimestamp}.json"
+    message_channel_identifier: "derivedlocationdata.state"
+    message_parser: "com.pinterest.secor.parser.ChannelDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  extractor-failed-backup:
+    replicas: 1
+    service_name: "extractor_failed_backup"
+    consumer_group: "{{ env_name }}.extractor.failed.backup"
+    base_path: "extractor-failed"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.extractor.failed"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 21600
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  assess-raw-events-backup:
+    replicas: 1
+    service_name: "assess_raw_events_backup"
+    consumer_group: "{{ env_name }}.telemetry.assess.raw"
+    base_path: "telemetry-raw-assess"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.assess.raw"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 21600
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  telemetry-duplicate-backup:
+    replicas: 1
+    service_name: "telemetry_duplicate_backup"
+    consumer_group: "{{ env_name }}.telemetry.duplicate.backup"
+    base_path: "telemetry-duplicate"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.duplicate"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  extractor-duplicate-backup:
+    replicas: 1
+    service_name: "extractor_duplicate_backup"
+    consumer_group: "{{ env_name }}.extractor.duplicate.backup"
+    base_path: "extractor-duplicate"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.extractor.duplicate"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  derived-telemetry-backup:
+    replicas: 1
+    service_name: "derived_telemetry_backup"
+    consumer_group: "{{ env_name }}.telemetry.derived.unique.backup"
+    base_path: "unique/summary"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.derived.unique"
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 600
+    partition_prefix_enabled: "true"
+    partition_prefix_key: "eid"
+    partition_prefix_mapping: '{"ME_WORKFLOW_SUMMARY":"workflow_summary","DEFAULT":"me"}'
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  device-profile-backup:
+    replicas: 1
+    service_name: "device_profile_backup"
+    consumer_group: "{{ env_name }}.events.device.profile.backup"
+    base_path: "device-profile-events"
+    timestamp_key: "updated_date"
+    fallback_timestamp_key: "updated_date"
+    topic: "{{ env_name }}.events.deviceprofile"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  learning-events-backup:
+    replicas: 1
+    service_name: "learning_events_backup"
+    consumer_group: "{{ env_name }}.learning.graph.events.backup"
+    base_path: "learning-events"
+    timestamp_key: "createdOn"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.learning.graph.events"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  learning-failed-backup:
+    replicas: 1
+    service_name: "learning_failed_backup"
+    consumer_group: "{{ env_name }}.failed.learning.events.backup"
+    base_path: "learning-failed-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    azure_account_name: "{{ secor.azure.account_name }}"
+    container_name: "{{ secor.azure.container_name }}"
+    azure_account_key: "{{ secor.azure.account_key }}"
+    topic: "{{ env_name }}.learning.events.failed"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 3600
+    partition_prefix_enabled: "true"
+    partition_prefix_key: "jobName"
+    partition_prefix_mapping: '{"publish-pipeline":"publish_pipeline","composite-search-indexer":"cs_index","DEFAULT":"failed_events"}'
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  assess-events-backup:
+    replicas: 1
+    service_name: "assess_events_backup"
+    consumer_group: "{{ env_name }}.telemetry.assess.events.backup"
+    base_path: "telemetry-batch-assess"
+    timestamp_key: "assessmentTs"
+    fallback_timestamp_key: "assessmentTs"
+    topic: "{{ env_name }}.telemetry.assess"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 21600
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  batch-assess-failed-events-backup:
+    replicas: 1
+    service_name: "batch_assess_failed_events_backup"
+    consumer_group: "{{ env_name }}.telemetry.assess.failed.events.backup"
+    base_path: "telemetry-batch-assess-failed"
+    timestamp_key: "assessmentTs"
+    fallback_timestamp_key: "assessmentTs"
+    topic: "{{ env_name }}.telemetry.assess.failed"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 21600
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000  
+  ingestion-telemetry-backup:
+    replicas: 1
+    service_name: "ingestion_telemetry_backup"
+    consumer_group: "{{ env_name }}.telemetry.ingestion.events.backup"
+    base_path: "ingestion-telemetry"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.telemetry.ingestion"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  content-consumption-events-backup:
+    replicas: 1
+    service_name: "content_consumption_events_backup"
+    consumer_group: "{{ env_name }}.coursebatch.job.request.backup"
+    base_path: "content-consumption-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.coursebatch.job.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  issue-certificate-events-backup:
+    replicas: 1
+    service_name: "issue_certificate_events_backup"
+    consumer_group: "{{ env_name }}.issue.certificate.request.backup"
+    base_path: "issue-certificate-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.issue.certificate.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  content-auto-creation-events-backup:
+    replicas: 1
+    service_name: "content_auto_creation_events_backup"
+    consumer_group: "{{ env_name }}.auto.creation.job.request.backup"
+    base_path: "content-auto-creation-events"
+    timestamp_key: "ets"
+    fallback_timestamp_key: "@timestamp"
+    topic: "{{ env_name }}.auto.creation.job.request"
+    kafka_broker_host: "{{ secor_ingestion_kafka_brokers }}"
+    zookeeper_quorum: "{{ ingestion_zookeepers }}"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
+  error-telemetry-backup:
+    replicas: 1
+    service_name: "error_events_backup"
+    consumer_group: '{{ env_name }}.druid.events.error.backup'
+    base_path: "error_events"
+    timestamp_key: "syncts"
+    fallback_timestamp_key: '@timestamp'
+    topic: '{{ env_name }}.druid.events.error'
+    kafka_broker_host: "{{groups['processing-cluster-kafka']|join(',')}}"
+    zookeeper_quorum: "{{groups['processing-cluster-zookeepers']|join(':2181,')}}:2181"
+    max_file_size: 100000000
+    max_file_age: 14400
+    partition_prefix_enabled: "false"
+    partition_prefix_key: ""
+    partition_prefix_mapping: "{}"
+    output_file_pattern: "{partition}-{currentTimestamp}.json"
+    message_channel_identifier: ""
+    message_parser: "com.pinterest.secor.parser.PatternDateMessageParser"
+    storage:
+      size: 10Gi
+    requests:
+      cpu: 500m
+      memory: 500Mi
+    lag_threshold_warning: 50000
+    lag_threshold_critical: 100000
\ No newline at end of file
diff --git a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml
new file mode 100644
index 0000000000..68778084ee
--- /dev/null
+++ b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml
@@ -0,0 +1,18 @@
+- name: template values.yaml file
+  vars:
+    secor_job_list: "{{ job_names_to_deploy.split(',')|list }}"
+  template:
+    src: "{{ chart_path }}/values.j2"
+    dest: "{{ chart_path }}/values.yaml"
+    
+- name: helm upgrade
+  shell: helm --debug upgrade --install {{ item }} {{ chart_path }} -n {{ secor_namespace }}
+  with_items:
+    - "{{ job_names_to_deploy.split(',')|list }}"
+  tags: secor_deploy
+
+- name: create alert rules
+  shell: helm  upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml
+  with_items:
+    - "{{ job_names_to_deploy.split(',')|list }}"
+  tags: alertrule
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml
index ddf8c0362c..5003f71a85 100644
--- a/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml
+++ b/kubernetes/ansible/roles/sunbird-monitoring/defaults/main.yml
@@ -4,6 +4,8 @@ fullnameOverride: sunbird-monitoring
 monitoring_stack:
   - prometheus-operator
   - alertrules
+  - prometheus-adapter
+  - processing-kafka-exporter
 
 namespace: monitoring
 flink_namespace: flink-dev
@@ -15,25 +17,57 @@ monitor_alerts_mail_server_host: "{{ sendgrid_hostname }}"
 monitor_alerts_mail_server_port: 587
 monitor_alerts_mail_server_username: "{{ SGUSER }}" 
 monitor_alerts_mail_server_password: "{{ SGPASS }}" 
-monitor_alerts_slack_url: 
-monitor_alerts_slack_channel: 
+dp_monitor_alerts_slack_url: 
+dp_monitor_alerts_slack_channel: "{{ env }}_flink_alerts"
+dp_monitor_alerts_warning_slack_url: 
+dp_monitor_alerts_warning_slack_channel: "{{ env }}_pipeline_warning_alerts"
+dp_monitor_alerts_critical_slack_url:
+dp_monitor_alerts_critical_slack_channel: "{{ env }}_pipeline_critical_alerts"
 
 default_critical_mailing_list: "{{ default_mailing_list }}"
 default_fatal_mailing_list: "{{ default_mailing_list }}"
 default_mailing_list: kaliraja.ramasamy@tarento.com, anandp@ilimi.in, manjunathd@ilimi.in
 
-# flink Jobs lag vars 
+# flink Jobs lag vars critical 
 telemetry_extractor_threshold_critical: 50000
 pipeline_preprocessor_threshold_critical: 50000
-de_normalization_threshold_critical: 50000
+de_normalization_threshold_critical: 50000   ### this var being used for de-normalization-primary and secondary jobs
 druid_validator_threshold_critical: 50000
 assessment_aggregator_threshold_critical: 50000
 content_cache_updater_threshold_critical: 50000
 user_cache_updater_threshold_critical: 50000
 summary_denormalization_threshold_critical: 50000
 device_profile_updater_threshold_critical: 50000
-telemetry_extractor_failed_events_percentage_threshold_fatal: 1
-telemetry_extractor_failed_events_percentage_threshold_critical: 0.5
+ingest_router_threshold_critical: 50000
+error_denormalization_threshold_critical: 50000
+de_normalization_primary_threshold_critical: 50000
+de_normalization_secondary_threshold_critical: 50000
+telemetry_extractor_failed_events_percentage_threshold_critical: 1
+telemetry_extractor_failed_events_percentage_threshold_warning: 0.5
+
+# Flink jobs lag warning
+telemetry_extractor_threshold_warning: 30000
+pipeline_preprocessor_threshold_warning: 30000
+de_normalization_threshold_warning: 30000 ### this var being used for de-normalization-primary and secondary jobs
+druid_validator_threshold_warning: 30000
+assessment_aggregator_threshold_warning: 30000
+content_cache_updater_threshold_warning: 30000
+user_cache_updater_threshold_warning: 30000
+summary_denormalization_threshold_warning: 30000
+device_profile_updater_threshold_warning: 30000
+ingest_router_threshold_warning: 30000
+error_denormalization_threshold_warning: 30000
+de_normalization_primary_threshold_warning: 30000
+de_normalization_secondary_threshold_warning: 30000
+# Flink KP Jobs lag vars critical
+activity_aggregater_updater_threshold_critical: 50
+relation_cache_updater_threshold_critical: 50
+post_publish_processor_threshold_critical: 50
+
+# Flink KP Jobs lag vars warning
+activity_aggregater_updater_threshold_warning: 40
+relation_cache_updater_threshold_warning: 40 
+post_publish_processor_threshold_warning: 40
 
 # Flink jobmanager Checkpoint failure vars
 telemetry_extractor_checkpointfailure_threshold_critical: 5 
@@ -45,3 +79,24 @@ content_cache_updater_checkpointfailure_threshold_critical: 5
 user_cache_updater_checkpointfailure_threshold_critical: 5
 summary_denormalization_checkpointfailure_threshold_critical: 5
 device_profile_updater_checkpointfailure_threshold_critical: 5
+user_cache_v2_updater_checkpointfailure_threshold_critical: 5
+relation_cache_updater_checkpointfailure_threshold_critical: 5
+activity_aggregate_updater_checkpointfailure_threshold_critical: 5
+ingest_router_checkpointfailure_threshold_critical: 5
+error_denormalization_checkpointfailure_threshold_critical: 5
+post_publish_processor_checkpontfailure_threshold_critical: 5
+
+nginx_tps_threshold: 50 # being used in flink jobs percentage lag alert rules
+flink_lag_percentage_threshold: 10 # being used in flink jobs percentage lag alert rule
+flink_lag_offset_duration: 30 # values in minutes and being used in alert rule
+
+pv_usage_threshold_critical: 90  # If the persistant volume used percentage is greater than 90% then will get an alert.
+kafka_exporter_image_tag: v1.3.0
+# kafka exporters vars
+### processing kafka exporters related vars
+processing_cluster_zookeeper: "{{ groups['processing-cluster-zookeepers'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}"
+processing_cluster_kafka: "{{ groups['processing-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}"
+
+### ingestion kafka exporters related vars.
+ingestion_cluster_zookeeper: "{{ groups['ingestion-cluster-zookeeper'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:2181') | list}}"
+ingestion_cluster_kafka: "{{ groups['ingestion-cluster-kafka'] | difference(['localhost']) | map('regex_replace', '^(.*)$', '\\1:9092') | list}}"
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml
index 6ce13d49d2..37f74a264a 100644
--- a/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml
+++ b/kubernetes/ansible/roles/sunbird-monitoring/tasks/main.yml
@@ -2,10 +2,10 @@
 # tasks file for sunbird-monitoring
 - name: tempating variables
   template:
-    src: "{{ item }}.yaml"
-    dest: "/tmp/{{item}}.yaml"
+    src: "dp_{{ item }}.yaml"
+    dest: "/tmp/dp_{{item}}.yaml"
   with_items: "{{ monitoring_stack }}"
 
 - name: Creating sunbird data infra monitoring stack
-  shell: "helm upgrade --install {{ item }} {{chart_path}}/{{ item }} --namespace monitoring -f /tmp/{{ item }}.yaml"
+  shell: "helm upgrade --install {{ item }} {{chart_path}}/{{ item }} --namespace monitoring -f /tmp/dp_{{ item }}.yaml"
   with_items: "{{ monitoring_stack }}"
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml
deleted file mode 100644
index ee7499d061..0000000000
--- a/kubernetes/ansible/roles/sunbird-monitoring/templates/alertrules.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-## This file contains alert rules related vars
-prometheus_rule_selector_app: prometheus-operator
-prometheus_rule_selector_release: prometheus-operator
-fullnameOverride: sunbird-monitoring
-namespace: monitoring
-
-# Node Exporter vars
-node_cpu_usage_percentage_threshold_Warning: 75
-node_cpu_usage_percentage_threshold_Critical: 85
-node_cpu_usage_percentage_threshold_Fatal: 95
-node_memory_usage_percentage_threshold_Critical: 85
-node_memory_usage_percentage_threshold_Warning: 75
-node_memory_usage_percentage_threshold_Fatal: 95
-node_load_avg_threshold_Critical: 95
-node_load_avg_threshold_Warning: 85
-node_load_avg_threshold_Fatal: 120
-node_disk_usage_percentage_threshold_Warning: 75
-node_disk_usage_percentage_threshold_Critical: 85
-node_disk_usage_percentage_threshold_Fatal: 95
-
-# Jobs lag vars 
-telemetry_extractor_threshold_critical: {{ telemetry_extractor_threshold_critical }}
-pipeline_preprocessor_threshold_critical: {{ pipeline_preprocessor_threshold_critical }}
-de_normalization_threshold_critical: {{ de_normalization_threshold_critical }}
-druid_validator_threshold_critical: {{ druid_validator_threshold_critical }}
-assessment_aggregator_threshold_critical: {{ assessment_aggregator_threshold_critical }}
-content_cache_updater_threshold_critical: {{ content_cache_updater_threshold_critical }}
-user_cache_updater_threshold_critical: {{ user_cache_updater_threshold_critical }}
-summary_denormalization_threshold_critical: {{ summary_denormalization_threshold_critical }}
-device_profile_updater_threshold_critical: {{ device_profile_updater_threshold_critical }}
-telemetry_extractor_failed_events_percentage_threshold_fatal: {{ telemetry_extractor_failed_events_percentage_threshold_fatal }}
-telemetry_extractor_failed_events_percentage_threshold_critical: {{ telemetry_extractor_failed_events_percentage_threshold_critical }}
-
-# Checkpoint failure vars
-telemetry_extractor_checkpointfailure_threshold_critical: {{ telemetry_extractor_checkpointfailure_threshold_critical }} 
-pipeline_preprocessor_checkpointfailure_threshold_critical: {{ pipeline_preprocessor_checkpointfailure_threshold_critical }}
-de_normalization_checkpointfailure_threshold_critical: {{ de_normalization_checkpointfailure_threshold_critical }}
-druid_validator_checkpointfailure_threshold_critical: {{ druid_validator_checkpointfailure_threshold_critical }}
-assessment_aggregator_checkpointfailure_threshold_critical: {{ assessment_aggregator_checkpointfailure_threshold_critical }}
-content_cache_updater_checkpointfailure_threshold_critical: {{ content_cache_updater_checkpointfailure_threshold_critical }}
-user_cache_updater_checkpointfailure_threshold_critical: {{ user_cache_updater_checkpointfailure_threshold_critical }}
-summary_denormalization_checkpointfailure_threshold_critical: {{ summary_denormalization_checkpointfailure_threshold_critical }}
-device_profile_updater_checkpointfailure_threshold_critical: {{ device_profile_updater_checkpointfailure_threshold_critical }}
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml
new file mode 100644
index 0000000000..a3cf54dd1f
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_additional-scrape-configs.yaml
@@ -0,0 +1,26 @@
+# This name is used to define the 
+# additionalScrapeConfigs name
+# {{ fullnameOverride }}-prometheus-scrape-confg
+# If you change this, make sure to update the value in 
+# additionalScrapeConfigs/defautls/main.yaml
+#}
+
+fullnameOverride: "sunbird-monitoring"
+
+scrapeconfig:
+  # This scrape config will enable us to annotate any pod and that pod will get automonitored
+  #
+  # annotations:
+  #   prometheus.io/scrape: "true"
+  #   prometheus.io/port: "2020"
+  #   prometheus.io/path: /api/v1/metrics/prometheus
+  #
+  - job_name: 'federate'
+    metrics_path: /prometheus/federate
+    honor_labels: true
+    params:
+      match[]:
+        - '{__name__=~"^job:.*"}'
+    static_configs:
+      # This empty line make sure the indentation is correct
+      - targets: [ '{{private_ingressgateway_ip}}' ]
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml
new file mode 100644
index 0000000000..071d20f694
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_alertrules.yaml
@@ -0,0 +1,95 @@
+## This file contains alert rules related vars
+prometheus_rule_selector_app: prometheus-operator
+prometheus_rule_selector_release: prometheus-operator
+fullnameOverride: sunbird-monitoring
+namespace: monitoring
+
+# Node Exporter vars
+node_cpu_usage_percentage_threshold_Warning: 80
+node_cpu_usage_percentage_threshold_Critical: 90
+node_cpu_usage_percentage_threshold_Fatal: 95
+node_memory_usage_percentage_threshold_Critical: 90
+node_memory_usage_percentage_threshold_Warning: 80
+node_memory_usage_percentage_threshold_Fatal: 95
+node_load_avg_threshold_Critical: 90
+node_load_avg_threshold_Warning: 80
+node_load_avg_threshold_Fatal: 95
+node_disk_usage_percentage_threshold_Warning: 80
+node_disk_usage_percentage_threshold_Critical: 90
+node_disk_usage_percentage_threshold_Fatal: 95
+
+# DP Jobs lag vars critical
+telemetry_extractor_threshold_critical: {{ telemetry_extractor_threshold_critical }}
+pipeline_preprocessor_threshold_critical: {{ pipeline_preprocessor_threshold_critical }}
+de_normalization_threshold_critical: {{ de_normalization_threshold_critical }}
+druid_validator_threshold_critical: {{ druid_validator_threshold_critical }}
+assessment_aggregator_threshold_critical: {{ assessment_aggregator_threshold_critical }}
+content_cache_updater_threshold_critical: {{ content_cache_updater_threshold_critical }}
+user_cache_updater_threshold_critical: {{ user_cache_updater_threshold_critical }}
+summary_denormalization_threshold_critical: {{ summary_denormalization_threshold_critical }}
+device_profile_updater_threshold_critical: {{ device_profile_updater_threshold_critical }}
+telemetry_extractor_failed_events_percentage_threshold_warning: {{ telemetry_extractor_failed_events_percentage_threshold_warning }}
+telemetry_extractor_failed_events_percentage_threshold_critical: {{ telemetry_extractor_failed_events_percentage_threshold_critical }}
+ingest_router_threshold_critical: {{ ingest_router_threshold_critical }}
+
+# DP Jobs lag vars warning
+telemetry_extractor_threshold_warning: {{ telemetry_extractor_threshold_warning }}
+pipeline_preprocessor_threshold_warning: {{ pipeline_preprocessor_threshold_warning }}
+de_normalization_threshold_warning: {{ de_normalization_threshold_warning }}
+druid_validator_threshold_warning: {{ druid_validator_threshold_warning }}
+assessment_aggregator_threshold_warning: {{ assessment_aggregator_threshold_warning }}
+content_cache_updater_threshold_warning: {{ content_cache_updater_threshold_warning }}
+user_cache_updater_threshold_warning: {{ user_cache_updater_threshold_warning }}
+summary_denormalization_threshold_warning: {{ summary_denormalization_threshold_warning }}
+device_profile_updater_threshold_warning: {{ device_profile_updater_threshold_warning }}
+ingest_router_threshold_warning: {{ ingest_router_threshold_warning }}
+
+
+# KP flink jobs lag vars critical
+activity_aggregater_updater_threshold_critical: {{ activity_aggregater_updater_threshold_critical }}
+relation_cache_updater_threshold_critical: {{ relation_cache_updater_threshold_critical }}
+post_publish_processor_threshold_critical: {{ post_publish_processor_threshold_critical }}
+
+# KP flink jobs lag vars warning
+activity_aggregater_updater_threshold_warning: {{ activity_aggregater_updater_threshold_warning }}
+relation_cache_updater_threshold_warning: {{ relation_cache_updater_threshold_warning }}
+post_publish_processor_threshold_warning: {{ post_publish_processor_threshold_warning }}
+
+# Checkpoint failure vars
+telemetry_extractor_checkpointfailure_threshold_critical: {{ telemetry_extractor_checkpointfailure_threshold_critical }} 
+pipeline_preprocessor_checkpointfailure_threshold_critical: {{ pipeline_preprocessor_checkpointfailure_threshold_critical }}
+de_normalization_checkpointfailure_threshold_critical: {{ de_normalization_checkpointfailure_threshold_critical }}
+druid_validator_checkpointfailure_threshold_critical: {{ druid_validator_checkpointfailure_threshold_critical }}
+assessment_aggregator_checkpointfailure_threshold_critical: {{ assessment_aggregator_checkpointfailure_threshold_critical }}
+content_cache_updater_checkpointfailure_threshold_critical: {{ content_cache_updater_checkpointfailure_threshold_critical }}
+user_cache_updater_checkpointfailure_threshold_critical: {{ user_cache_updater_checkpointfailure_threshold_critical }}
+summary_denormalization_checkpointfailure_threshold_critical: {{ summary_denormalization_checkpointfailure_threshold_critical }}
+device_profile_updater_checkpointfailure_threshold_critical: {{ device_profile_updater_checkpointfailure_threshold_critical }}
+user_cache_v2_updater_checkpointfailure_threshold_critical: {{ user_cache_v2_updater_checkpointfailure_threshold_critical }}
+relation_cache_updater_checkpointfailure_threshold_critical: {{ relation_cache_updater_checkpointfailure_threshold_critical }}
+activity_aggregate_updater_checkpointfailure_threshold_critical: {{ activity_aggregate_updater_checkpointfailure_threshold_critical }}
+
+
+flink_job_names:
+  assessment-aggregator: {{ assessment_aggregator_threshold_critical }} 
+  content-cache-updater: {{ content_cache_updater_threshold_critical }} 
+  de-normalization-primary: {{ de_normalization_primary_threshold_critical }}
+  de-normalization-secondary: {{ de_normalization_secondary_threshold_critical }}
+  device-profile-updater: {{ device_profile_updater_threshold_critical }}  
+  druid-validator: {{ druid_validator_threshold_critical }} 
+  error-denormalization: {{ error_denormalization_threshold_critical }}
+  ingest-router: {{ ingest_router_threshold_critical }} 
+  pipeline-preprocessor: {{ pipeline_preprocessor_threshold_critical }}
+  summary-denormalization: {{ summary_denormalization_threshold_critical }}
+  telemetry-extractor: {{ telemetry_extractor_threshold_critical }}       
+  user-cache-updater-v2: {{ user_cache_updater_threshold_critical }}
+## KP flink jobs
+  activity-aggregate-updater: {{ activity_aggregater_updater_threshold_critical }}
+  relation-cache-updater: {{ relation_cache_updater_threshold_critical }}
+  post-publish-processor: {{ post_publish_processor_threshold_critical }}
+
+nginx_tps_threshold: {{ nginx_tps_threshold }} # being used in flink jobs percentage lag alertrules
+flink_lag_percentage_threshold: {{ flink_lag_percentage_threshold }} # being used in flink jobs percentage lag alertrules
+flink_lag_offset_duration: {{ flink_lag_offset_duration }} # values in mins.  offset time window to compare the metrics between current metrics and last 30(example) minutes. 
+
+pv_usage_threshold_critical: {{ pv_usage_threshold_critical }}
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml
new file mode 100644
index 0000000000..2956b3976f
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_ingestion-kafka-exporter.yaml
@@ -0,0 +1,22 @@
+kafkaExporter:
+  zookeeper:
+    servers: ["{{ ingestion_cluster_zookeeper | join('","') }}"]
+  kafka:
+    servers: ["{{ ingestion_cluster_kafka | join('","') }}"]
+  additionalFlags:
+    - --use.consumelag.zookeeper
+
+image:
+  repository: danielqsj/kafka-exporter
+  tag: "{{ kafka_exporter_image_tag }}" 
+  pullPolicy: IfNotPresent
+
+prometheus:
+  serviceMonitor:
+    enabled: true
+    namespace: monitoring
+    interval: "120s"
+    scrapeTimeout: "90s"
+    additionalLabels:
+      app: prometheus-operator
+      release: prometheus-operator
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml
new file mode 100644
index 0000000000..1f03467f34
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_processing-kafka-exporter.yaml
@@ -0,0 +1,22 @@
+kafkaExporter:
+  zookeeper:
+    servers: ["{{ processing_cluster_zookeeper | join('","') }}"]
+  kafka:
+    servers: ["{{ processing_cluster_kafka | join('","') }}"]
+  additionalFlags:
+    - --use.consumelag.zookeeper
+
+image:
+  repository: danielqsj/kafka-exporter
+  tag: "{{ kafka_exporter_image_tag }}"
+  pullPolicy: IfNotPresent
+
+prometheus:
+  serviceMonitor:
+    enabled: true
+    namespace: monitoring
+    interval: "120s"
+    scrapeTimeout: "90s"
+    additionalLabels:
+      app: prometheus-operator
+      release: prometheus-operator
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml
new file mode 100644
index 0000000000..74a8add5ea
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-adapter.yaml
@@ -0,0 +1,146 @@
+# Default values for k8s-prometheus-adapter..
+
+# Url to access prometheus
+prometheus:
+  url: http://sunbird-monitoring-prometheus.monitoring.svc.cluster.local
+  port: 9090
+  path: ""
+
+rules:
+  default: true
+  custom: {}
+  existing:
+  external:
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "telemetry-extractor_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-extractor-group",topic="{{ env_name }}.telemetry.ingest"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "ingest-router_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-ingest-router-group"}) by (consumergroup)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "pipeline-preprocessor_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-pipeline-preprocessor-group",topic="{{ env_name }}.telemetry.raw"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "de-normalization-primary_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-denorm-primary-group",topic="{{ env_name }}.telemetry.unique.primary"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "de-normalization-secondary_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-telemetry-denorm-secondary-group",topic="{{ env_name }}.telemetry.unique.secondary"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "druid-validator_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-druid-validator-group",topic="{{ env_name }}.telemetry.denorm"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "error-denormalization_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-error-denorm-group",topic="{{ env_name }}.telemetry.error"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "summary-denormalization_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-summmary-denorm-group",topic="{{ env_name }}.telemetry.derived"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "content-cache-updater_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-content-cache-updater-group",topic="{{ env_name }}.learning.graph.events"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "device-profile-updater_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-device-profile-updater-group",topic="{{ env_name }}.events.deviceprofile"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "user-cache-updater-v2_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-user-cache-updater-consumer-group",topic="{{ env_name }}.telemetry.audit"}) by (<<.GroupBy>>)
+  - seriesQuery: 'kafka_consumergroup_lag_sum'
+    resources:
+      overrides:
+        namespace:
+          resource: namespace
+        pod:
+          resource: pod
+    name:
+      matches: "^(.*)"
+      as: "assessment-aggregator_kafka_consumergroup_lag_sum"
+    metricsQuery: sum(kafka_consumergroup_lag_sum{consumergroup="{{ env_name }}-assessment-aggregator-group",topic="{{ env_name }}.telemetry.assess"}) by (<<.GroupBy>>)
+  
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml
new file mode 100644
index 0000000000..2fb4f89192
--- /dev/null
+++ b/kubernetes/ansible/roles/sunbird-monitoring/templates/dp_prometheus-operator.yaml
@@ -0,0 +1,242 @@
+#jinja2:lstrip_blocks: True
+{#
+# This name is used to define the 
+# additionalScrapeConfigs name
+# {{ fullnameOverride }}-prometheus-scrape-confg
+# If you change this, make sure to update the value in 
+# additionalScrapeConfigs/defautls/main.yaml
+#}
+fullnameOverride: sunbird-monitoring
+
+# Enabling external prometheus scrape config 
+prometheus:
+  prometheusSpec:
+    additionalScrapeConfigsExternal: {{ additional_scrape_configs_enabled | d(false) }}
+    retention: "{{ prometheus_retention_time | d('60d') }}"
+    externalLabels:
+      sb_cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}"
+{% if prometheus_storage_spec is defined and prometheus_storage_spec %}
+    storageSpec: {{ prometheus_storage_spec|to_json }}
+{% endif %}
+{% if prometheus_service is defined and prometheus_service %}
+  service: {{ (prometheus_service | to_json) }}
+{% endif %}
+kubeControllerManager: 
+  enabled: false
+kubeScheduler:
+  enabled: false
+
+
+alertmanager:
+  config:
+    global:
+      smtp_from: "{{ monitor_alerts_mail_from_email }}"
+      smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}"
+      smtp_auth_username: "{{ monitor_alerts_mail_server_username }}"
+      smtp_auth_password: "{{ monitor_alerts_mail_server_password }}"
+    route:
+      receiver: '{{env}}_devops_team'
+      group_by: [sb_cluster, alertname]
+      group_wait: 30s
+      group_interval: 5m
+      repeat_interval: 4h
+      routes:
+      - match_re:
+         alertname: ^(node_exporter_down_critical|Watchdog|PrometheusRuleFailures)
+        receiver: "null"
+      - match:
+          alerttype: nodemetrics
+          severity: warning
+        receiver: nodemetrics_slack_warning
+      - match:
+          alerttype: nodemetrics
+          severity: critical
+        receiver: nodemetrics_slack_critical
+      - match:
+          alerttype: lag
+          severity: warning
+        receiver: lag_slack_warning
+      - match:
+          alerttype: lag
+          severity: critical
+        receiver: lag_slack_critical
+      - match:
+          alerttype: job
+          severity: critical
+        receiver: jobs_slack_critical
+      - receiver: slack
+        continue: true
+      - match:
+          owner:
+        receiver: {{env}}_devops_team
+    receivers:
+      - name: {{env}}_devops_team
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+      - name: 'jobs_slack_critical'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_critical_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_critical_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }}
+                    *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }}
+                    *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }}
+                    *Details:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+
+      - name: 'lag_slack_warning'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_warning_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_warning_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }}
+                    *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }}
+                    *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }}
+                    *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }}
+                    *Details:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+        
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+
+      - name: 'lag_slack_critical'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_critical_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_critical_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }}
+                    *Job:* {{ "{{" }} .Annotations.job_id {{ "}}" }}
+                    *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }}
+                    *AlertMetric:* {{ "{{" }} .Annotations.lag {{ "}}" }}
+                    *Details:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+        
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+        
+      - name: 'nodemetrics_slack_warning'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_warning_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_warning_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }}
+                    *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }}
+                    *AlertMetric:* {{ "{{" }} .Annotations.metrics {{ "}}" }}
+                    *Details:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+
+      - name: 'nodemetrics_slack_critical'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_critical_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_critical_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.alertname {{ "}}" }}
+                    *AlertType:* {{ "{{" }} .Labels.severity {{ "}}" }}
+                    *AlertMetric:* {{ "{{" }} .Annotations.metrics {{ "}}" }}
+                    *Details:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+
+        email_configs:
+          - send_resolved: true
+            to: '{{ default_mailing_list }}'
+            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
+            headers:
+              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
+
+
+      - name: 'slack'
+        slack_configs:
+          - send_resolved: true
+            api_url: "{{ dp_monitor_alerts_warning_slack_url }}"
+            username: 'Monitor - Alerter'
+            channel: "{{ dp_monitor_alerts_warning_slack_channel }}"
+            title_link: ""
+            title: '{% raw %}[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}]{% endraw %}' 
+            text: |-
+                  {{ "{{" }} range .Alerts {{ "}}" }}
+                    *Alert:* {{ "{{" }} .Annotations.message {{ "}}" }} - `{{ "{{" }} .Labels.severity {{ "}}" }}`
+                    *Description:* {{ "{{" }} .Annotations.message {{ "}}" }}
+                    *Details:*
+                    {{ "{{" }} range .Labels.SortedPairs {{ "}}" }} • *{{ "{{" }} .Name {{ "}}" }}:* `{{ "{{" }} .Value {{ "}}" }}`
+                    {{ "{{" }} end {{ "}}" }}
+                  {{ "{{" }} end {{ "}}" }}
+            icon_emoji: ':dart:'
+
+      - name: 'null'
+
+grafana:
+  env:
+    #    GF_SERVER_ROOT_URL: http://grafana.local.com/grafana
+  adminPassword: {{ (grafana_admin_password| default('prom-operator'))}}
+{% if grafana_data_sources is defined and grafana_data_sources %}
+  additionalDataSources: {{ (grafana_data_sources) | to_json}}
+{% endif %}
+{% if grafana_persistence is defined and grafana_persistence %}
+  persistence: {{ grafana_persistence|to_json}}
+{% endif %}
+{% if grafana_service is defined and grafana_service %}
+  service: {{ grafana_service|to_json}}
+{% endif %}
+ 
+## Overriding the kubernetes-apps alert rules by disabling default alert rule creation. 
+# Why: We were getting few unwanted alerts and we wanted to disable those alerts. Hence created the custom alert rules.
+defaultRules:
+  create: true
+  rules:
+    kubernetesApps: false
diff --git a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml b/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml
deleted file mode 100644
index f624aad38f..0000000000
--- a/kubernetes/ansible/roles/sunbird-monitoring/templates/prometheus-operator.yaml
+++ /dev/null
@@ -1,71 +0,0 @@
-#jinja2:lstrip_blocks: True
-{#
-# This name is used to define the 
-# additionalScrapeConfigs name
-# {{ fullnameOverride }}-prometheus-scrape-confg
-# If you change this, make sure to update the value in 
-# additionalScrapeConfigs/defautls/main.yaml
-#}
-fullnameOverride: sunbird-monitoring
-
-# Enabling external prometheus scrape config 
-prometheus:
-  prometheusSpec:
-    additionalScrapeConfigsExternal: {{ additional_scrape_configs_enabled | d(false) }}
-    retention: "{{ prometheus_retention_time | d('90d') }}"
-    externalLabels:
-      sb_cluster: "{{ kubernetes_cluster_name | default('kubernetes-1')}}"
-{% if prometheus_storage_spec is defined and prometheus_storage_spec %}
-    storageSpec: {{ prometheus_storage_spec|to_json }}
-{% endif %}
-{% if prometheus_service is defined and prometheus_service %}
-  service: {{ (prometheus_service | to_json) }}
-{% endif %}
-
-
-alertmanager:
-  config:
-    global:
-      smtp_from: "{{ monitor_alerts_mail_from_email }}"
-      smtp_smarthost: "{{ monitor_alerts_mail_server_host }}:{{ monitor_alerts_mail_server_port}}"
-      smtp_auth_username: "{{ monitor_alerts_mail_server_username }}"
-      smtp_auth_password: "{{ monitor_alerts_mail_server_password }}"
-    route:
-      receiver: '{{env}}_devops_team'
-      group_by: [sb_cluster, alertname]
-      group_wait: 30s
-      group_interval: 5m
-      repeat_interval: 4h
-      routes:
-      - match:
-          owner:
-        receiver: {{env}}_devops_team
-    receivers:
-      - name: {{env}}_devops_team
-        email_configs:
-          - send_resolved: true
-            to: '{{ default_mailing_list }}'
-            html: '{% raw %}{{ template "email.default.html" . }}{% endraw %}'
-            headers:
-              subject: '[{{ kubernetes_cluster_name }}] {% raw %}{{ .GroupLabels.alertname }}{% endraw %}'
-
-grafana:
-  env:
-    #    GF_SERVER_ROOT_URL: http://grafana.local.com/grafana
-  adminPassword: {{ (grafana_admin_password| default('prom-operator'))}}
-{% if grafana_data_sources is defined and grafana_data_sources %}
-  additionalDataSources: {{ (grafana_data_sources) | to_json}}
-{% endif %}
-{% if grafana_persistence is defined and grafana_persistence %}
-  persistence: {{ grafana_persistence|to_json}}
-{% endif %}
-{% if grafana_service is defined and grafana_service %}
-  service: {{ grafana_service|to_json}}
-{% endif %}
- 
-## Overriding the kubernetes-apps alert rules by disabling default alert rule creation. 
-# Why: We were getting few unwanted alerts and we wanted to disable those alerts. Hence created the custom alert rules.
-defaultRules:
-  create: true
-  rules:
-    kubernetesApps: false
diff --git a/kubernetes/ansible/secor_deploy.yaml b/kubernetes/ansible/secor_deploy.yaml
new file mode 100644
index 0000000000..15de8ddd4b
--- /dev/null
+++ b/kubernetes/ansible/secor_deploy.yaml
@@ -0,0 +1,20 @@
+---
+- hosts: local
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    KUBECONFIG: "{{ kubeconfig_path }}"
+  roles:
+    - secor-deploy
+  tags: secor_deploy
+
+- hosts: local
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  environment:
+    KUBECONFIG: "{{ core_kubeconfig_path }}"
+  roles:
+    - secor-deploy
+  tags: alertrule
diff --git a/kubernetes/helm_charts/bootstrap/reloader/.helmignore b/kubernetes/helm_charts/bootstrap/reloader/.helmignore
new file mode 100755
index 0000000000..9e1690881f
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/.helmignore
@@ -0,0 +1,2 @@
+# OWNERS file for Kubernetes
+OWNERS
diff --git a/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml b/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml
new file mode 100755
index 0000000000..b4995a9a82
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/Chart.yaml
@@ -0,0 +1,27 @@
+apiVersion: v1
+appVersion: v0.0.49
+description: Reloader chart that runs on kubernetes
+home: https://github.com/stakater/Reloader
+icon: https://raw.githubusercontent.com/stakater/Reloader/master/assets/web/reloader-round-100px.png
+keywords:
+- Reloader
+- kubernetes
+maintainers:
+- email: hello@stakater.com
+  name: Stakater
+- email: rasheed@aurorasolutions.io
+  name: rasheedamir
+- email: waseemhassan@stakater.com
+  name: waseem-h
+- email: faizan.ahmad55@outlook.com
+  name: faizanahmad055
+- email: ali.kahoot@aurorasolutions.io
+  name: kahootali
+- email: ahmad@aurorasolutions.io
+  name: ahmadiq
+- email: ahsanmuhammad1@outlook.com
+  name: ahsan-storm
+name: reloader
+sources:
+- https://github.com/stakater/IngressMonitorController
+version: v0.0.49
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt b/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt
new file mode 100755
index 0000000000..f2a38752a4
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/NOTES.txt
@@ -0,0 +1,7 @@
+- For a `Deployment` called `foo` have a `ConfigMap` called `foo-configmap`. Then add this annotation to main metadata of your `Deployment`
+  configmap.reloader.stakater.com/reload: "foo-configmap"
+
+- For a `Deployment` called `foo` have a `Secret` called `foo-secret`. Then add this annotation to main metadata of  your `Deployment`
+  secret.reloader.stakater.com/reload: "foo-secret"
+
+- After successful installation, your pods will get rolling updates when a change in data of configmap or secret will happen.
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl b/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl
new file mode 100755
index 0000000000..15ca27ef7d
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/_helpers.tpl
@@ -0,0 +1,35 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+
+{{- define "reloader-name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" | lower -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "reloader-fullname" -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "reloader-labels.chart" -}}
+app: {{ template "reloader-fullname" . }}
+chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+release: {{ .Release.Name | quote }}
+heritage: {{ .Release.Service | quote }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "reloader-serviceAccountName" -}}
+{{- if .Values.reloader.serviceAccount.create -}}
+    {{ default (include "reloader-fullname" .) .Values.reloader.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.reloader.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml
new file mode 100755
index 0000000000..8d51ef406b
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrole.yaml
@@ -0,0 +1,62 @@
+{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  labels:
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.rbac.labels }}
+{{ toYaml .Values.reloader.rbac.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-fullname" . }}-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+{{- if .Values.reloader.ignoreSecrets }}{{- else }}
+      - secrets
+{{- end }}
+{{- if .Values.reloader.ignoreConfigMaps }}{{- else }}
+      - configmaps
+{{- end }}
+    verbs:
+      - list
+      - get
+      - watch
+{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.isOpenshift) }}
+  - apiGroups:
+      - "apps.openshift.io"
+      - ""
+    resources:
+      - deploymentconfigs
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+{{- end }}
+  - apiGroups:
+      - "apps"
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - deployments
+      - daemonsets
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+{{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml
new file mode 100755
index 0000000000..28c9d4b916
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/clusterrolebinding.yaml
@@ -0,0 +1,23 @@
+{{- if and .Values.reloader.watchGlobally (.Values.reloader.rbac.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.rbac.labels }}
+{{ toYaml .Values.reloader.rbac.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-fullname" . }}-role-binding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "reloader-fullname" . }}-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "reloader-serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml
new file mode 100755
index 0000000000..c03a86f50e
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/deployment.yaml
@@ -0,0 +1,129 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+{{- if .Values.reloader.deployment.annotations }}
+  annotations:
+{{ toYaml .Values.reloader.deployment.annotations | indent 4 }}
+{{- end }}
+  labels:
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.deployment.labels }}
+{{ toYaml .Values.reloader.deployment.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-fullname" . }}
+spec:
+  replicas: 1
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: {{ template "reloader-fullname" . }}
+      release: {{ .Release.Name | quote }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 6 }}
+{{- end }}
+  template:
+    metadata:
+{{- if .Values.reloader.deployment.pod.annotations }}
+      annotations:
+{{ toYaml .Values.reloader.deployment.pod.annotations | indent 8 }}
+{{- end }}
+      labels:
+{{ include "reloader-labels.chart" . | indent 8 }}
+{{- if .Values.reloader.deployment.labels }}
+{{ toYaml .Values.reloader.deployment.labels | indent 8 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 8 }}
+{{- end }}
+    spec:
+      {{- if .Values.reloader.deployment.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.reloader.deployment.nodeSelector | indent 8 }}
+      {{- end }}
+      {{- if .Values.reloader.deployment.affinity }}
+      affinity:
+{{ toYaml .Values.reloader.deployment.affinity | indent 8 }}
+      {{- end }}
+      {{- if .Values.reloader.deployment.tolerations }}
+      tolerations:
+{{ toYaml .Values.reloader.deployment.tolerations | indent 8 }}
+      {{- end }}
+      containers:
+      - env:
+      {{- range $name, $value := .Values.reloader.deployment.env.open }}
+      {{- if not (empty $value) }}
+        - name: {{ $name | quote }}
+          value: {{ $value | quote }}
+      {{- end }}
+      {{- end }}
+      {{- $secret_name := include "reloader-fullname" . }}
+      {{- range $name, $value := .Values.reloader.deployment.env.secret }}
+      {{- if not ( empty $value) }}
+        - name: {{ $name | quote }}
+          valueFrom:
+            secretKeyRef:
+              name: {{ $secret_name }}
+              key: {{ $name | quote }}
+      {{- end }}
+      {{- end }}
+      {{- range $name, $value := .Values.reloader.deployment.env.field }}
+      {{- if not ( empty $value) }}
+        - name: {{ $name | quote }}
+          valueFrom:
+            fieldRef:
+              fieldPath: {{ $value | quote}}
+      {{- end }}
+      {{- end }}
+      {{- if eq .Values.reloader.watchGlobally false }}
+        - name: KUBERNETES_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+      {{- end }}
+        image: "{{ .Values.reloader.deployment.image.name }}:{{ .Values.reloader.deployment.image.tag }}"
+        imagePullPolicy: {{ .Values.reloader.deployment.image.pullPolicy }}
+        name: {{ template "reloader-fullname" . }}
+      {{- if eq .Values.reloader.readOnlyRootFileSystem true }}
+        volumeMounts:
+          - mountPath: /tmp/
+            name: tmp-volume
+      {{- end }}
+        args:
+          {{- if .Values.reloader.ignoreSecrets }}
+          - "--resources-to-ignore=secrets"
+          {{- end }}
+          {{- if eq .Values.reloader.ignoreConfigMaps true }}
+          - "--resources-to-ignore=configMaps"
+          {{- end }}
+
+          {{- if .Values.reloader.custom_annotations }}
+            {{- if .Values.reloader.custom_annotations.configmap }}
+            - "--configmap-annotation"
+            - "{{ .Values.reloader.custom_annotations.configmap }}"
+            {{- end }}
+            {{- if .Values.reloader.custom_annotations.secret }}
+            - "--secret-annotation"
+            - "{{ .Values.reloader.custom_annotations.secret }}"
+            {{- end }}
+            {{- if .Values.reloader.custom_annotations.auto }}
+            - "--auto-annotation"
+            - "{{ .Values.reloader.custom_annotations.auto }}"
+            {{- end }}
+          {{- end }}
+
+      {{- if .Values.reloader.deployment.resources }}
+        resources:
+{{ toYaml .Values.reloader.deployment.resources | indent 10 }}
+      {{- end }}
+{{- if .Values.reloader.deployment.securityContext }}
+      securityContext: {{ toYaml .Values.reloader.deployment.securityContext | nindent 8 }}
+{{- end }}
+      serviceAccountName: {{ template "reloader-serviceAccountName" . }}
+    {{- if eq .Values.reloader.readOnlyRootFileSystem true }}
+      volumes:
+        - emptyDir: {}
+          name: tmp-volume
+    {{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml
new file mode 100755
index 0000000000..5827f5cdcb
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/role.yaml
@@ -0,0 +1,62 @@
+{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: Role
+metadata:
+  labels:
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.rbac.labels }}
+{{ toYaml .Values.reloader.rbac.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-fullname" . }}-role
+  namespace: {{ .Release.Namespace }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+{{- if .Values.reloader.ignoreSecrets }}{{- else }}
+      - secrets
+{{- end }}
+{{- if .Values.reloader.ignoreConfigMaps }}{{- else }}
+      - configmaps
+{{- end }}
+    verbs:
+      - list
+      - get
+      - watch
+{{- if or (.Capabilities.APIVersions.Has "apps.openshift.io/v1") (.Values.reloader.isOpenshift) }}
+  - apiGroups:
+      - "apps.openshift.io"
+      - ""
+    resources:
+      - deploymentconfigs
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+{{- end }}
+  - apiGroups:
+      - "apps"
+    resources:
+      - deployments
+      - daemonsets
+      - statefulsets
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+  - apiGroups:
+      - "extensions"
+    resources:
+      - deployments
+      - daemonsets
+    verbs:
+      - list
+      - get
+      - update
+      - patch
+{{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml
new file mode 100755
index 0000000000..94fb1f838b
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/rolebinding.yaml
@@ -0,0 +1,23 @@
+{{- if and (not (.Values.reloader.watchGlobally)) (.Values.reloader.rbac.enabled) }}
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: RoleBinding
+metadata:
+  labels: 
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.rbac.labels }}
+{{ toYaml .Values.reloader.rbac.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-fullname" . }}-role-binding
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ template "reloader-fullname" . }}-role
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "reloader-serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml b/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml
new file mode 100755
index 0000000000..62e5f1a0af
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/templates/serviceaccount.yaml
@@ -0,0 +1,17 @@
+{{- if .Values.reloader.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+{{- if .Values.global.imagePullSecrets }}
+imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }}
+{{- end }}
+metadata:
+  labels:
+{{ include "reloader-labels.chart" . | indent 4 }}
+{{- if .Values.reloader.serviceAccount.labels }}
+{{ toYaml .Values.reloader.serviceAccount.labels | indent 4 }}
+{{- end }}
+{{- if .Values.reloader.matchLabels }}
+{{ toYaml .Values.reloader.matchLabels | indent 4 }}
+{{- end }}
+  name: {{ template "reloader-serviceAccountName" . }}
+{{- end }}
diff --git a/kubernetes/helm_charts/bootstrap/reloader/values.yaml b/kubernetes/helm_charts/bootstrap/reloader/values.yaml
new file mode 100755
index 0000000000..0973025cbd
--- /dev/null
+++ b/kubernetes/helm_charts/bootstrap/reloader/values.yaml
@@ -0,0 +1,88 @@
+# Generated from deployments/kubernetes/templates/chart/values.yaml.tmpl
+global:
+  ## Reference to one or more secrets to be used when pulling images
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ##
+  imagePullSecrets: []
+
+kubernetes:
+  host: https://kubernetes.default
+
+reloader:
+  isOpenshift: false
+  ignoreSecrets: false
+  ignoreConfigMaps: false
+  watchGlobally: true
+  # Set to true if you have a pod security policy that enforces readOnlyRootFilesystem
+  readOnlyRootFileSystem: false
+  matchLabels: {}
+  deployment:
+    nodeSelector:
+    # cloud.google.com/gke-nodepool: default-pool
+
+    # An affinity stanza to be applied to the Deployment.
+    # Example:
+    #   affinity:
+    #     nodeAffinity:
+    #       requiredDuringSchedulingIgnoredDuringExecution:
+    #         nodeSelectorTerms:
+    #         - matchExpressions:
+    #           - key: "node-role.kubernetes.io/infra-worker"
+    #             operator: "Exists"
+    affinity: {}
+
+    # A list of tolerations to be applied to the Deployment.
+    # Example:
+    #   tolerations:
+    #   - key: "node-role.kubernetes.io/infra-worker"
+    #     operator: "Exists"
+    #     effect: "NoSchedule"
+    tolerations: []
+
+    labels:
+      provider: stakater
+      group: com.stakater.platform
+      version: v0.0.49
+    image:
+      name: stakater/reloader
+      tag: "v0.0.49"
+      pullPolicy: IfNotPresent
+    # Support for extra environment variables.
+    env:
+      # Open supports Key value pair as environment variables.
+      open:
+      # secret supports Key value pair as environment variables. It gets the values based on keys from default reloader secret if any.
+      secret:
+      # field supports Key value pair as environment variables. It gets the values from other fields of pod.
+      field:
+
+    # Specify resource requests/limits for the deployment.
+    # Example:
+    # resources:
+    #   limits:
+    #     cpu: "100m"
+    #     memory: "512Mi"
+    #   requests:
+    #     cpu: "10m"
+    #     memory: "128Mi"
+    resources: {}
+    pod:
+      annotations: {}
+
+  rbac:
+    enabled: true
+    labels: {}
+  # Service account config for the agent pods
+  serviceAccount:
+    # Specifies whether a ServiceAccount should be created
+    create: true
+    labels: {}
+    # The name of the ServiceAccount to use.
+    # If not set and create is true, a name is generated using the fullname template
+    name: 
+  # Optional flags to pass to the Reloader entrypoint
+  # Example:
+  #   custom_annotations:
+  #     configmap: "my.company.com/configmap"
+  #     secret: "my.company.com/secret"
+  custom_annotations: {}
diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml
index 4c0201904f..48a2719ab6 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml
+++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_configmap.yaml
@@ -8,6 +8,7 @@ metadata:
 data:
   base-config: |+
 {{ .Values.base_config | indent 4 }}
-  {{ .Release.Name }}:
 {{- $name := .Release.Name }}
-{{ index .Values $name | toYaml | indent 4 }}
+{{ index .Values $name | toYaml | indent 2 }}
+  log4j_console_properties: |+
+{{ .Values.log4j_console_properties | indent 4 }}
diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
index 89c40c6b15..10e6b62181 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
+++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml
@@ -84,16 +84,21 @@ spec:
         component: {{ .Release.Name }}-jobmanager
       annotations:
         prometheus.io/scrape: 'true'
+        prometheus.io/port: "{{ .Values.jobmanager.prom_port }}"
     spec:
       volumes:
       - name: flink-config-volume
         configMap:
           name: {{ .Release.Name }}-config
           items:
+          - key: flink-conf
+            path: flink-conf.yaml
           - key: base-config
             path: base-config.conf
           - key: {{ .Release.Name }}
             path: {{ .Release.Name }}.conf
+          - key: log4j_console_properties
+            path: log4j-console.properties
       restartPolicy: OnFailure
       imagePullSecrets:
       - name: {{ .Values.imagepullsecrets }}
@@ -104,8 +109,16 @@ spec:
         workingDir: /opt/flink
         command: ["/opt/flink/bin/standalone-job.sh"]
         args: ["start-foreground",
-               "--job-classname={{ .Values.job_classname }}", 
+               "--job-classname={{ .Values.job_classname }}",
+{{- if eq .Values.checkpoint_store_type "azure" }}
                "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "s3" }}
+               "-Ds3.access-key={{ .Values.s3_access_key }}",
+               "-Ds3.secret-key={{ .Values.s3_secret_key }}",
+               "-Ds3.endpoint={{ .Values.s3_endpoint }}",
+               "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
                "-Dweb.submit.enable=false",
                "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
                "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}",
@@ -114,7 +127,6 @@ spec:
                "-Dparallelism.default=1", 
                "-Dblob.server.port={{ .Values.jobmanager.blob_port }}", 
                "-Dqueryable-state.server.ports={{ .Values.jobmanager.query_port }}",
-               "-Djobmanager.heap.size={{ .Values.jobmanager.heap_memory }}m",
                "--config.file.path",
                "/data/flink/conf/{{ .Release.Name }}.conf"]
         ports:
@@ -128,7 +140,17 @@ spec:
           name: ui
         volumeMounts:
         - name: flink-config-volume
-          mountPath: /data/flink/conf
+          mountPath: /opt/flink/conf/flink-conf.yaml
+          subPath: flink-conf.yaml
+        - name: flink-config-volume
+          mountPath: /data/flink/conf/base-config.conf
+          subPath: base-config.conf
+        - name: flink-config-volume
+          mountPath: /data/flink/conf/{{ .Release.Name }}.conf
+          subPath: {{ .Release.Name }}.conf
+        - name: flink-config-volume
+          mountPath: /opt/flink/conf/log4j-console.properties
+          subPath: log4j-console.properties
 
 ---
 apiVersion: apps/v1
@@ -148,23 +170,42 @@ spec:
         app: flink
         component: {{ .Release.Name }}-taskmanager
     spec:
+      volumes:
+      - name: flink-config-volume
+        configMap:
+          name: {{ .Release.Name }}-config
+          items:
+          - key: flink-conf
+            path: flink-conf.yaml
+          - key: log4j_console_properties
+            path: log4j-console.properties 
       imagePullSecrets:
       - name: {{ .Values.imagepullsecrets }}
       containers:
       - name: {{ .Release.Name }}-taskmanager
         image: "{{ .Values.dockerhub }}/{{ .Values.repository }}:{{ .Values.image_tag }}"
         imagePullPolicy: Always
+        resources:
+          requests:
+            cpu: "{{ .Values.taskmanager.cpu_requests }}"
         workingDir: {{ .Values.taskmanager.flink_work_dir }}
         command: ["/opt/flink/bin/taskmanager.sh"]
         args: ["start-foreground",
+{{- if eq .Values.checkpoint_store_type "azure" }}
           "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+{{- end }}
+{{- if eq .Values.checkpoint_store_type "s3" }}
+          "-Ds3.access-key={{ .Values.s3_access_key }}",
+          "-Ds3.secret-key={{ .Values.s3_secret_key }}",
+          "-Ds3.endpoint={{ .Values.s3_endpoint }}",
+          "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
+{{- end }}
           "-Dweb.submit.enable=false",
           "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
           "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager",
           "-Dmetrics.reporter.prom.port=9251-9260",
           "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
-          "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}",
-          "-Dtaskmanager.heap.size={{ .Values.taskmanager.heap_memory }}m"]
+          "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"]
         ports:
         - containerPort: {{ .Values.taskmanager.rpc_port }}
           name: rpc
@@ -172,3 +213,33 @@ spec:
         livenessProbe:
 {{ toYaml .Values.livenessProbe | indent 10 }}
         {{- end }}
+        volumeMounts:
+        - name: flink-config-volume
+          mountPath: /opt/flink/conf/flink-conf.yaml
+          subPath: flink-conf.yaml
+        - name: flink-config-volume
+          mountPath: /opt/flink/conf/log4j-console.properties
+          subPath: log4j-console.properties
+
+{{- $name := .Release.Name }}
+{{- $prop := (index .Values.scale_properties $name)}}
+{{- if $prop.enabled}}
+---
+apiVersion: autoscaling/v2beta1
+kind: HorizontalPodAutoscaler
+metadata:
+  name: {{ .Release.Name }}-taskmanager-hpa
+  namespace: {{ .Values.namespace }}
+spec:
+  scaleTargetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: {{ .Release.Name }}-taskmanager
+  minReplicas: {{ $prop.min_replica }}
+  maxReplicas: {{ $prop.max_replica }}
+  metrics:
+    - type: External
+      external:
+        metricName: {{ .Release.Name }}_kafka_consumergroup_lag_sum
+        targetValue: "{{ $prop.scale_target_value }}"
+{{- end }}        
\ No newline at end of file
diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml
index 76cbf60215..6b6d0c2dda 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml
+++ b/kubernetes/helm_charts/datapipeline_jobs/templates/serviceMonitor.yaml
@@ -1,3 +1,4 @@
+{{- if .Values.serviceMonitor.enabled }}
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -36,3 +37,4 @@ spec:
   namespaceSelector:
     matchNames:
     - {{ .Values.namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2
index f7867d2ef1..22630c3015 100644
--- a/kubernetes/helm_charts/datapipeline_jobs/values.j2
+++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2
@@ -3,11 +3,79 @@ imagepullsecrets: {{ imagepullsecrets }}
 dockerhub: {{ dockerhub }}
 repository: {{flink_repository|default('sunbird-datapipeline')}}
 image_tag: {{ image_tag }}
-registry: {{ docker_registry }}
+checkpoint_store_type: {{ checkpoint_store_type }}
 azure_account: {{ azure_account }}
 azure_secret: {{ azure_secret }}
+s3_access_key: {{ s3_storage_key }}
+s3_secret_key: {{ s3_storage_secret }}
+s3_endpoint: {{ s3_storage_endpoint }}
+s3_path_style_access: {{ s3_path_style_access }}
 
-replicaCount: {{taskmana_replicacount|default(1)}}
+serviceMonitor:
+  enabled: {{ service_monitor_enabled | lower}}
+replicaCount: {{taskmanager_replicacount|default(1)}}
+
+scale_properties:
+  telemetry-extractor:
+    enabled: {{ flink_job_names['telemetry-extractor'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['telemetry-extractor'].scale_target_value }}
+    min_replica: {{ flink_job_names['telemetry-extractor'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['telemetry-extractor'].max_replica | default(2) }}
+  pipeline-preprocessor:
+    enabled: {{ flink_job_names['pipeline-preprocessor'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['pipeline-preprocessor'].scale_target_value }}
+    min_replica: {{ flink_job_names['pipeline-preprocessor'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['pipeline-preprocessor'].max_replica | default(2) }}
+  de-normalization-secondary:
+    enabled: {{ flink_job_names['de-normalization-secondary'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['de-normalization-secondary'].scale_target_value }} 
+    min_replica: {{ flink_job_names['de-normalization-secondary'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['de-normalization-secondary'].max_replica | default(2) }}
+  de-normalization-primary:
+    enabled: {{ flink_job_names['de-normalization-primary'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['de-normalization-primary'].scale_target_value }}
+    min_replica: {{ flink_job_names['de-normalization-primary'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['de-normalization-primary'].max_replica | default(2) }}      
+  druid-validator:
+    enabled: {{ flink_job_names['druid-validator'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['druid-validator'].scale_target_value }}
+    min_replica: {{ flink_job_names['druid-validator'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['druid-validator'].max_replica | default(2) }}
+  assessment-aggregator:
+    enabled: {{ flink_job_names['assessment-aggregator'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['assessment-aggregator'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['assessment-aggregator'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['assessment-aggregator'].max_replica | default(2) }}
+  content-cache-updater:
+    enabled: {{ flink_job_names['content-cache-updater'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['content-cache-updater'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['content-cache-updater'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['content-cache-updater'].max_replica | default(2) }}
+  user-cache-updater-v2:
+    enabled: {{ flink_job_names['user-cache-updater-v2'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['user-cache-updater-v2'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['user-cache-updater-v2'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['user-cache-updater-v2'].max_replica | default(2) }}
+  summary-denormalization:
+    enabled: {{ flink_job_names['summary-denormalization'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['summary-denormalization'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['summary-denormalization'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['summary-denormalization'].max_replica | default(2) }}
+  device-profile-updater:
+    enabled: {{ flink_job_names['device-profile-updater'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['device-profile-updater'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['device-profile-updater'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['device-profile-updater'].max_replica | default(2) }}
+  ingest-router:
+    enabled: {{ flink_job_names['ingest-router'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['ingest-router'].scale_target_value }}
+    min_replica: {{ flink_job_names['ingest-router'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['ingest-router'].max_replica | default(2) }}
+  error-denormalization:
+    enabled: {{ flink_job_names['error-denormalization'].scale_enabled | lower}}
+    scale_target_value: {{ flink_job_names['error-denormalization'].scale_target_value | default(0) }}
+    min_replica: {{ flink_job_names['error-denormalization'].min_replica | default(1) }}
+    max_replica: {{ flink_job_names['error-denormalization'].max_replica | default(2) }}
 
 jobmanager:
   rpc_port: {{ jobmanager_rpc_port }}
@@ -27,17 +95,53 @@ taskmanager:
   rpc_port: {{ taskmanager_rpc_port }}
   heap_memory: {{ taskmanager_heap_memory }}
   replicas: {{taskmanager_replicacount|default(1)}}
+  cpu_requests: {{ cpu_requests|default(1)}}
 
 job_classname: {{ job_classname }}
 {{ taskmanager_liveness | to_nice_yaml }}
 
+log4j_console_properties: |
+  # This affects logging for both user code and Flink
+  rootLogger.level = {{ flink_jobs_console_log_level | default(INFO) }}
+  rootLogger.appenderRef.console.ref = ConsoleAppender
+
+  # Uncomment this if you want to _only_ change Flink's logging
+  #logger.flink.name = org.apache.flink
+  #logger.flink.level = {{ flink_jobs_console_log_level | default(INFO) }}
+
+  # The following lines keep the log level of common libraries/connectors on
+  # log level INFO. The root logger does not override this. You have to manually
+  # change the log levels here.
+  logger.akka.name = akka
+  logger.akka.level = {{ flink_libraries_log_level | default(INFO) }}
+  logger.kafka.name= org.apache.kafka
+  logger.kafka.level = {{ flink_libraries_log_level | default(INFO) }}
+  logger.hadoop.name = org.apache.hadoop
+  logger.hadoop.level = {{ flink_libraries_log_level | default(INFO) }}
+  logger.zookeeper.name = org.apache.zookeeper
+  logger.zookeeper.level = {{ flink_libraries_log_level | default(INFO) }}
+
+  # Log all infos to the console
+  appender.console.name = ConsoleAppender
+  appender.console.type = CONSOLE
+  appender.console.layout.type = PatternLayout
+  appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss,SSS} %-5p %-60c %x - %m%n
+
+  # Suppress the irrelevant (wrong) warnings from the Netty channel handler
+  logger.netty.name = org.apache.flink.shaded.akka.org.jboss.netty.channel.DefaultChannelPipeline
+  logger.netty.level = OFF
 
 base_config: |
   kafka {
       broker-servers = "{{ kafka_brokers }}"
+      producer.broker-servers = "{{ kafka_brokers }}"
+      consumer.broker-servers = "{{ kafka_brokers }}"
       zookeeper = "{{ zookeepers }}"
       producer {
         max-request-size = {{ producer_max_request_size }}
+        batch.size = {{ producer_batch_size }}
+        linger.ms = {{ producer_linger_ms }}
+        compression = {{ producer_compression }}
       }
     }
     job {
@@ -46,20 +150,24 @@ base_config: |
       statebackend {
         blob {
           storage {
-            account = "{{ azure_account }}.blob.core.windows.net"
+            account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% endif %}"
             container = "{{ flink_container_name }}"
             checkpointing.dir = "checkpoint"
           }
         }
+{% if checkpoint_store_type == "azure" %}
         base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
+{% elif checkpoint_store_type == "s3" %}
+        base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
+{% endif %}
       }
     }
     task {
       parallelism = 1
-      consumer {
-        parallelism = 1
-      }
+      consumer.parallelism = 1
+      checkpointing.compressed = {{ checkpoint_compression_enabled|lower }}
       checkpointing.interval = {{ checkpoint_interval }}
+      checkpointing.pause.between.seconds = {{ checkpoint_pause_between_seconds }}
       restart-strategy.attempts = {{ restart_attempts }}
       restart-strategy.delay = {{ restart_delay }} # in milli-seconds
     }
@@ -69,8 +177,8 @@ base_config: |
       port = 6379
     }
     redis-meta {
-{% if metadata_redis_host is defined %}
-      host = {{ metadata_redis_host }}
+{% if metadata2_redis_host is defined %}
+      host = {{ metadata2_redis_host }}
 {% else %}
       host = {{ redis_host }}
 {% endif %}
@@ -88,269 +196,583 @@ base_config: |
       port = "9042"
     }
 
-telemetry-extractor: | 
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.topic = {{ env_name }}.telemetry.ingest
-    output.success.topic = {{ env_name }}.telemetry.raw
-    output.duplicate.topic = {{ env_name }}.telemetry.extractor.duplicate
-    output.failed.topic = {{ env_name }}.telemetry.failed
-    output.batch.failed.topic = {{ env_name }}.telemetry.extractor.failed
-    output.assess.raw.topic = {{ env_name }}.telemetry.assess.raw
-    event.max.size = "{{ extractor_event_max_size }}" # Max is only 1MB
-    groupId = {{ env_name }}-telemetry-extractor-group
-    producer {
-      max-request-size = {{ extractor_max_request_size }}
-    }
-  }
-  task {
-    consumer.parallelism = {{ extractor_consumer_parallelism }}
-    dedup.parallelism = {{ dedup_parallelism }}
-    extraction.parallelism = {{ extraction_parallelism }}
-    redactor.parallelism = {{ redactor_parallelism }}
-  }
-  redis {
-    database {
-      duplicationstore.id = 1
-      key.expiry.seconds = {{ telemetry_extractor_key_expiry_seconds }} 
-      contentstore.id = 5
-    }
-  }
-  redis-meta {
+ingest-router:
+  ingest-router: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      producer.broker-servers = "{{ kafka_brokers }}"
+      consumer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      ingest {
+       input.topic = {{ env_name }}.telemetry.ingestion
+       output.success.topic = {{ env_name }}.telemetry.ingest
+      }
+      raw {
+       input.topic = "{{ env_name }}.telemetry.raw"
+       output.success.topic = "{{ env_name }}.telemetry.raw"
+      }
+      groupId = {{ env_name }}-ingest-router-group
+      producer {
+        max-request-size = {{ extractor_max_request_size }}
+      }
+    }
+    task {
+      consumer.parallelism = {{ ingest_router_consumer_parallelism }}
+      downstream.operators.parallelism = {{ ingest_router_operators_parallelism }}
+      raw {
+       consumer.parallelism = {{ raw_router_consumer_parallelism }}
+       downstream.operators.parallelism = {{ raw_router_downstream_parallelism }}
+      }
+    }
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['ingest-router'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['ingest-router'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['ingest-router'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['ingest-router'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['ingest-router'].jobmanager_process_memory }}
+
+
+telemetry-extractor:
+  telemetry-extractor: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.topic = {{ env_name }}.telemetry.ingest
+      output.success.topic = {{ env_name }}.telemetry.raw
+      output.log.route.topic = {{ env_name }}.druid.events.log
+      output.duplicate.topic = {{ env_name }}.telemetry.extractor.duplicate
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.batch.failed.topic = {{ env_name }}.telemetry.extractor.failed
+      output.assess.raw.topic = {{ env_name }}.telemetry.assess.raw
+      event.max.size = "{{ extractor_event_max_size }}" # Max is only 1MB
+      groupId = {{ env_name }}-telemetry-extractor-group
+      producer {
+        max-request-size = {{ extractor_max_request_size }}
+      }
+    }
+    task {
+      consumer.parallelism = {{ extractor_consumer_parallelism }}
+      downstream.operators.parallelism = {{ extractor_operators_parallelism }}
+    }
+    redis {
       database {
+        duplicationstore.id = 1
+        key.expiry.seconds = {{ telemetry_extractor_key_expiry_seconds }} 
         contentstore.id = 5
       }
     }
-  redact.events.list = ["ASSESS","RESPONSE"]
+    redis-meta {
+        database {
+          contentstore.id = 5
+        }
+        host = {{ redis_meta_content_host }}
+        port = {{ redis_meta_content_port }}
+      }
+    redact.events.list = ["ASSESS","RESPONSE"]
 
-pipeline-preprocessor: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.topic = {{ env_name }}.telemetry.raw
-    output.failed.topic = {{ env_name }}.telemetry.failed
-    output.primary.route.topic = {{ env_name }}.telemetry.unique
-    output.log.route.topic = {{ env_name }}.druid.events.log
-    output.error.route.topic = {{ env_name }}.druid.events.error
-    output.audit.route.topic = {{ env_name }}.telemetry.audit
-    output.duplicate.topic = {{ env_name }}.telemetry.duplicate
-    groupId = {{ env_name }}-pipeline-preprocessor-group
-  }
-  task {
-    consumer.parallelism = {{ pipeline_preprocessor_consumer_parallelism }}
-    telemetry.validation.parallelism = {{ telemetry_validation_parallelism }}
-    telemetry.router.parallelism = {{ telemetry_router_parallelism }}
-    share.events.flattener.parallelism = {{ share_events_flattener_parallelism }}
-  }
-  telemetry.schema.path="schemas/telemetry/3.0"
-  default.channel="b00bc992ef25f1a9a8d63291e20efc8d"
-  dedup.producer.included.ids = ["{{ portal_id }}", "{{ desktop_id }}"]
-  redis {
-    database {
-      duplicationstore.id = 7
-      key.expiry.seconds = {{ pipeline_preprocessor_key_expiry_seconds }}
-    }
-  }
-
-de-normalization: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.telemetry.topic = {{ env_name }}.telemetry.unique
-    input.summary.topic = {{ env_name }}.telemetry.derived
-    output.success.topic = {{ env_name }}.telemetry.denorm
-    output.failed.topic = {{ env_name }}.telemetry.failed
-    output.duplicate.topic = {{ env_name }}.telemetry.duplicate
-    output.summary.topic = {{ env_name }}.telemetry.derived.unique
-    groupId = {{ env_name }}-telemetry-denorm-group
-  }
-  task {
-    consumer.parallelism = {{ denorm_consumer_parallelism }}
-    denorm.parallelism = {{ denorm_parallelism }}
-    denorm.sink.parallelism = {{ denorm_sink_parallelism }}
-    denorm.summary-dedup.parallelism = {{ denorm_summary_dedup_parallelism }}
-    summary.sink.parallelism = {{ summary_sink_parallelism }}
-  }
-  redis {
-    database {
-      duplicationstore.id = 9
-      key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} 
-    }
-  }
-  # redis-metadata
-  redis-meta {
-    database {
-      devicestore.id = 2
-      userstore.id = 4
-      contentstore.id = 5
-      dialcodestore.id = 6
-      key.expiry.seconds = {{ de_normalization_key_expiry_seconds }}
-    }
-  }
-
-summary-denormalization: |+
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.telemetry.topic = {{ env_name }}.telemetry.unique
-    input.summary.topic = {{ env_name }}.telemetry.derived
-    output.success.topic = {{ env_name }}.telemetry.denorm
-    output.failed.topic = {{ env_name }}.telemetry.failed
-    output.duplicate.topic = {{ env_name }}.telemetry.duplicate
-    output.summary.topic = {{ env_name }}.telemetry.derived.unique
-    groupId = {{ env_name }}-summmary-denorm-group
-  }
-  task {
-    consumer.parallelism = {{ summary_denorm_consumer_parallelism }}
-    denorm.parallelism = {{ summary_denorm_parallelism }}
-    denorm.sink.parallelism = {{ summary_denorm_sink_parallelism }}
-    denorm.summary-dedup.parallelism = {{ summary_denorm_dedup_parallelism }}
-    summary.sink.parallelism = {{ summary_denorm_summary_sink_parallelism }}
-  }
-  redis {
-    database {
-      duplicationstore.id = 9
-      key.expiry.seconds = {{ summary_denorm_duplication_key_expiry_seconds }}
-    }
-  }
-  # redis-metadata
-  redis-meta {
-    database {
-      devicestore.id = 2
-      userstore.id = 4
-      contentstore.id = 5
-      dialcodestore.id = 6
-      key.expiry.seconds = {{ summary_denorm_key_expiry_seconds }}
-    }
-  }
-
-druid-validator: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.topic = {{ env_name }}.telemetry.denorm
-    output.telemetry.route.topic = {{ env_name }}.druid.events.telemetry
-    output.summary.route.topic = {{ env_name }}.druid.events.summary
-    output.failed.topic = {{ env_name }}.telemetry.failed
-    output.duplicate.topic = {{ env_name }}.telemetry.duplicate
-    groupId = {{ env_name }}-druid-validator-group
-  }
-  task {
-    consumer.parallelism = {{ druid_validator_consumer_parallelism }}
-    validator.parallelism = {{ validator_parallelism }} 
-    router.parallelism = {{ router_parallelism }}
-    druid.validation.enabled = {{ druid_validation_enabled|lower }}
-    druid.deduplication.enabled = {{ druid_deduplication_enabled|lower }}
-  }
-  schema {
-    path {
-      telemetry = "schemas/telemetry"
-      summary = "schemas/summary"
-    }
-    file {
-      default = envelope.json
-      summary = me_workflow_summary.json
-      search = search.json
-    }
-  }
-  redis {
-    database {
-      duplicationstore.id = 8
-      key.expiry.seconds = {{ druid_validator_key_expiry_seconds }}
-    }
-  }
-
-device-profile-updater: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    broker-servers = "{{ ingestion_kafka_brokers }}"
-    zookeeper = "{{ ingestion_zookeepers }}"
-    input.topic = {{ env_name }}.events.deviceprofile
-    groupId = {{ env_name }}-device-profile-updater-group
-  }
-  task {
-    deviceprofile {
-      parallelism = {{ deviceprofile_parallelism }}
-    }
-  }
-  # redis-metadata
-  redis-meta {
-    host = {{ metadata_redis_host }}
-    port = 6379
-    database {
-      devicestore.id = 2
-    }
-  }
-  postgres {
-    database = "{{ postgres.db_name }}",
-    table = "{{ device_profile_table }}"
-  }
-
-content-cache-updater: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    broker-servers = "{{ ingestion_kafka_brokers }}"
-    zookeeper = "{{ ingestion_zookeepers }}"
-    input.topic = {{ env_name }}.learning.graph.events
-    groupId = {{ env_name }}-content-cache-updater-group
-  }
-  # redis-metadata
-  redis-meta {
-    host = {{ metadata_redis_host }}
-    port = 6379
-    database {
-      contentstore.id = 5
-      dialcodestore.id = 6
-    }
-  }
-  dialcode {
-    api {
-      url = "{{ dialcode_api_url }}"
-      token = "{{ dialcode_api_auth_key }} "
-    }
-  }
-
-user-cache-updater: |
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    input.topic = {{ env_name }}.telemetry.audit
-    groupId = {{ env_name }}-user-cache-updater-group
-  }
-  task {
-    usercache.updater.parallelism = {{ usercache_updater_parallelism }}
-  }
-  # redis-metadata
-  redis-meta {
-    host = {{ metadata_redis_host }}
-    port = 6379
-    database {
-      userstore.id = 4
-    }
-  }
-  lms-cassandra {
-    keyspace = "{{ middleware_cassandra_keyspace }}"
-    table {
-      user = "{{ middleware_cassandra_user_table }}"
-      location = "{{ middleware_cassandra_location_table }}"
-    }
-  }
-  user.self.signin.types = ["google","self"]
-  user.validated.types = ["sso"]
-  user.self.signin.key = "Self-Signed-In"
-  user.valid.key = "Validated"
-
-assessment-aggregator: |+
-  include file("/data/flink/conf/base-config.conf")
-  kafka {
-    broker-servers = "{{ ingestion_kafka_brokers }}"
-    zookeeper = "{{ ingestion_zookeepers }}"
-    input.topic = {{ env_name }}.telemetry.assess
-    failed.topic= {{ env_name }}.telemetry.assess.failed
-    groupId = {{ env_name }}-assessment-aggregator-group
-  }
-  task {
-    assessaggregator {
-      parallelism = {{ assessaggregator_parallelism }}
-    }
-  }
-  lms-cassandra {
-    keyspace = "{{ middleware_cassandra_courses_keyspace }}"
-    table = "{{ middleware_cassandra_assessment_aggregator_table }}"
-    questionudttype= "{{ middleware_cassandra_assessment_question_type }}"
-  }
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['telemetry-extractor'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['telemetry-extractor'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['telemetry-extractor'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['telemetry-extractor'].jobmanager_process_memory }}
+
+pipeline-preprocessor:
+  pipeline-preprocessor: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.topic = {{ env_name }}.telemetry.raw
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.primary.route.topic = {{ env_name }}.telemetry.unique
+      output.log.route.topic = {{ env_name }}.druid.events.log
+      output.error.route.topic = {{ env_name }}.telemetry.error
+      output.audit.route.topic = {{ env_name }}.telemetry.audit
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      output.denorm.secondary.route.topic = {{ env_name }}.telemetry.unique.secondary
+      output.denorm.primary.route.topic = {{ env_name }}.telemetry.unique.primary
+      groupId = {{ env_name }}-pipeline-preprocessor-group
+    }
+    task {
+      consumer.parallelism = {{ pipeline_preprocessor_consumer_parallelism }}
+      downstream.operators.parallelism = {{ pipeline_preprocessor_operators_parallelism }}
+    }
+    telemetry.schema.path="schemas/telemetry/3.0"
+    default.channel="b00bc992ef25f1a9a8d63291e20efc8d"
+    dedup.producer.included.ids = ["{{ portal_id }}", "{{ desktop_id }}"]
+    secondary.events = ["INTERACT", "IMPRESSION", "SHARE_ITEM"]
+    redis {
+      database {
+        duplicationstore.id = 7
+        key.expiry.seconds = {{ pipeline_preprocessor_key_expiry_seconds }}
+      }
+    }
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['pipeline-preprocessor'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['pipeline-preprocessor'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['pipeline-preprocessor'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['pipeline-preprocessor'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['pipeline-preprocessor'].jobmanager_process_memory }}
+
+de-normalization-secondary:
+  de-normalization-secondary: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.telemetry.topic = {{ env_name }}.telemetry.unique.secondary
+      input.summary.topic = {{ env_name }}.telemetry.derived
+      telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm
+      summary.denorm.output.topic = {{ env_name }}.druid.events.summary
+      summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      groupId = {{ env_name }}-telemetry-denorm-secondary-group
+    }
+    skip.events = ["INTERRUPT"]
+    permit.eid=["AUDIT"]
+    task {
+      window.count = {{ denorm_secondary_window_count }}
+      window.shards = {{ denorm_secondary_window_shards }}
+      consumer.parallelism = {{ denorm_secondary_consumer_parallelism }}
+      telemetry.downstream.operators.parallelism = {{ telemetry_denorm_secondary_operators_parallelism }}
+      summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism  }}
+    }
+    redis {
+      database {
+        duplicationstore.id = 9
+        key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} 
+      }
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        devicestore.id = 2
+        userstore.id = 12
+        contentstore.id = 5
+        dialcodestore.id = 6
+        key.expiry.seconds = {{ de_normalization_key_expiry_seconds }}
+      }
+      content.host = {{ redis_meta_content_host }}
+      device.host = {{ redis_meta_device_host }}
+      user.host = {{ redis_meta_user_host }}
+      dialcode.host = {{ redis_meta_dialcode_host }}
+      content.port = {{ redis_meta_content_port }}
+      device.port = {{ redis_meta_device_port }}
+      user.port = {{ redis_meta_user_port }}
+      dialcode.port = {{ redis_meta_dialcode_port }}
+    }
+    # config version
+    user_denorm_version = v2
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['de-normalization-secondary'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['de-normalization-secondary'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['de-normalization-secondary'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['de-normalization-secondary'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['de-normalization-secondary'].jobmanager_process_memory }}
+
+de-normalization-primary:
+  de-normalization-primary: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.telemetry.topic = {{ env_name }}.telemetry.unique.primary
+      input.summary.topic = {{ env_name }}.telemetry.derived
+      telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm
+      summary.denorm.output.topic = {{ env_name }}.druid.events.summary
+      summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      groupId = {{ env_name }}-telemetry-denorm-primary-group
+    }
+    skip.events = ["INTERRUPT"]
+    permit.eid=["AUDIT"]
+    task {
+      window.count = {{ denorm_primary_window_count }}
+      window.shards = {{ denorm_primary_window_shards }}
+      consumer.parallelism = {{ denorm_primary_consumer_parallelism }}
+      telemetry.downstream.operators.parallelism = {{ telemetry_denorm_primary_operators_parallelism }}
+      summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism  }}
+    }
+    redis {
+      database {
+        duplicationstore.id = 9
+        key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} 
+      }
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        devicestore.id = 2
+        userstore.id = 12
+        contentstore.id = 5
+        dialcodestore.id = 6
+        key.expiry.seconds = {{ de_normalization_key_expiry_seconds }}
+      }
+      content.host = {{ redis_meta_content_host }}
+      device.host = {{ redis_meta_device_host }}
+      user.host = {{ redis_meta_user_host }}
+      dialcode.host = {{ redis_meta_dialcode_host }}
+      content.port = {{ redis_meta_content_port }}
+      device.port = {{ redis_meta_device_port }}
+      user.port = {{ redis_meta_user_port }}
+      dialcode.port = {{ redis_meta_dialcode_port }}
+    }
+    # config version
+    user_denorm_version = v2
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['de-normalization-primary'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['de-normalization-primary'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['de-normalization-primary'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['de-normalization-primary'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['de-normalization-primary'].jobmanager_process_memory }}
+
+summary-denormalization:
+  summary-denormalization: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.telemetry.topic = {{ env_name }}.telemetry.unique
+      input.summary.topic = {{ env_name }}.telemetry.derived
+      telemetry.denorm.output.topic = {{ env_name }}.telemetry.denorm
+      summary.denorm.output.topic = {{ env_name }}.druid.events.summary
+      summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      groupId = {{ env_name }}-summmary-denorm-group
+    }
+    skip.events = ["INTERRUPT"]
+    permit.eid=["AUDIT"]
+    task {
+      window.count = {{ denorm_summary_window_count }}
+      window.shards = {{ denorm_summary_window_shards }}
+      consumer.parallelism = {{ summary_denorm_consumer_parallelism }}
+      telemetry.downstream.operators.parallelism = {{ telemetry_denorm_operators_parallelism }}
+      summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism  }}
+    }
+    redis {
+      database {
+        duplicationstore.id = 9
+        key.expiry.seconds = {{ summary_denorm_duplication_key_expiry_seconds }}
+      }
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        devicestore.id = 2
+        userstore.id = 12
+        contentstore.id = 5
+        dialcodestore.id = 6
+        key.expiry.seconds = {{ summary_denorm_key_expiry_seconds }}
+      }
+      content.host = {{ redis_meta_content_host }}
+      device.host = {{ redis_meta_device_host }}
+      user.host = {{ redis_meta_user_host }}
+      dialcode.host = {{ redis_meta_dialcode_host }}
+      content.port = {{ redis_meta_content_port }}
+      device.port = {{ redis_meta_device_port }}
+      user.port = {{ redis_meta_user_port }}
+      dialcode.port = {{ redis_meta_dialcode_port }}
+    }
+    # config version
+    user_denorm_version = v2
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['summary-denormalization'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['summary-denormalization'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['summary-denormalization'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['summary-denormalization'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['summary-denormalization'].jobmanager_process_memory }}
+
+druid-validator:
+  druid-validator: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.topic = {{ env_name }}.telemetry.denorm
+      output.telemetry.route.topic = {{ env_name }}.druid.events.telemetry
+      output.summary.route.topic = {{ env_name }}.druid.events.summary
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      groupId = {{ env_name }}-druid-validator-group
+    }
+    task {
+      consumer.parallelism = {{ druid_validator_consumer_parallelism }}
+      downstream.operators.parallelism = {{ druid_validator_operators_parallelism }}
+      druid.validation.enabled = {{ druid_validation_enabled|lower }}
+      druid.deduplication.enabled = {{ druid_deduplication_enabled|lower }}
+    }
+    schema {
+      path {
+        telemetry = "schemas/telemetry"
+        summary = "schemas/summary"
+      }
+      file {
+        default = envelope.json
+        summary = me_workflow_summary.json
+        search = search.json
+      }
+    }
+    redis {
+      database {
+        duplicationstore.id = 8
+        key.expiry.seconds = {{ druid_validator_key_expiry_seconds }}
+      }
+    }
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['druid-validator'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['druid-validator'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['druid-validator'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+    scheduler-mode: reactive
+    heartbeat.timeout: 8000
+    heartbeat.interval: 5000
+    taskmanager.memory.process.size: {{ flink_job_names['druid-validator'].taskmanager_process_memory }}
+    jobmanager.memory.process.size: {{ flink_job_names['druid-validator'].jobmanager_process_memory }}
+
+device-profile-updater:
+  device-profile-updater: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      producer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      consumer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      zookeeper = "{{ ingestion_zookeepers }}"
+      input.topic = {{ env_name }}.events.deviceprofile
+      groupId = {{ env_name }}-device-profile-updater-group
+    }
+    task {
+      deviceprofile {
+        parallelism = {{ deviceprofile_parallelism }}
+      }
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        devicestore.id = 2
+      }
+      host = {{ redis_meta_device_host }}
+      port = {{ redis_meta_device_port }}
+    }
+    postgres {
+      database = "{{ postgres.db_name }}",
+      table = "{{ device_profile_table }}"
+    }
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['device-profile-updater'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['device-profile-updater'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['device-profile-updater'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+
+content-cache-updater:
+  content-cache-updater: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      producer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      consumer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      zookeeper = "{{ ingestion_zookeepers }}"
+      input.topic = {{ env_name }}.learning.graph.events
+      groupId = {{ env_name }}-content-cache-updater-group
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        contentstore.id = 5
+        dialcodestore.id = 6
+      }
+      content.host = {{ redis_meta_content_host }}
+      dialcode.host = {{ redis_meta_dialcode_host }}
+      content.port = {{ redis_meta_content_port }}
+      dialcode.port = {{ redis_meta_dialcode_port }}
+    }
+    dialcode {
+      api {
+        url = "{{ dialcode_api_url }}"
+        token = "{{ sunbird_api_auth_token }} "
+      }
+    }
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['content-cache-updater'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['content-cache-updater'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['content-cache-updater'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+
+user-cache-updater-v2:
+  user-cache-updater-v2: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.topic = {{ env_name }}.telemetry.audit
+      groupId = {{ env_name }}-user-cache-updater-consumer-group
+    }
+    task {
+      usercache.updater.parallelism = {{ usercache_updater_parallelism }}
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        userstore.id = 12
+      }
+      host = {{ redis_meta_user_host }}
+      port = {{ redis_meta_user_port }}
+    }
+    user-read {
+     api {
+       url = "{{ user_read_api_url  }}"
+     }
+    }
+    regd.user.producer.pid = "learner-service"
+    user.self.signin.types = ["google","self"]
+    user.validated.types = ["sso"]
+    user.self.signin.key = "Self-Signed-In"
+    user.valid.key = "Validated"
+    user.read.url.fields = "locations,organisations"
+    user.read.api.error = ["CLIENT_ERROR"]
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['user-cache-updater-v2'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['user-cache-updater-v2'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['user-cache-updater-v2'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+
+assessment-aggregator: 
+  assessment-aggregator: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      producer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      consumer.broker-servers = "{{ ingestion_kafka_brokers }}"
+      zookeeper = "{{ ingestion_zookeepers }}"
+      input.topic = {{ env_name }}.telemetry.assess
+      failed.topic= {{ env_name }}.telemetry.assess.failed
+      groupId = {{ env_name }}-assessment-aggregator-group
+      output.certissue.topic = {{ env_name }}.issue.certificate.request
+    }
+    task {
+      consumer.parallelism = {{ assessaggregator_consumer_parallelism }}
+      downstream.parallelism = {{ assessaggregator_downstream_parallelism }}
+      assessaggregator {
+        parallelism = {{ assessaggregator_parallelism }}
+      }
+      scoreaggregator.parallelism = {{ assessaggregator_scoreaggregator_parallelism }}
+    }
+    lms-cassandra {
+      keyspace = "{{ middleware_cassandra_courses_keyspace }}"
+      table = "{{ middleware_cassandra_assessment_aggregator_table }}"
+      questionudttype= "{{ middleware_cassandra_assessment_question_type }}"
+      enrolmentstable = "{{ middleware_cassandra_user_enrolments_table }}"
+      activitytable = "{{ middleware_cassandra_user_activity_agg_table }}"
+    }
+    redis {
+      database {
+        relationCache.id = 10
+        contentCache.id = 5
+      }
+    }
+    assessment.skip.missingRecords = true
+    content.read.api = "{{ content_read_api_host }}/{{ content_read_api_endpoint }}"
+    user.activity.agg.type="attempt_metrics"
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['assessment-aggregator'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['assessment-aggregator'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['assessment-aggregator'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
+
+error-denormalization:
+  error-denormalization: |+
+    include file("/data/flink/conf/base-config.conf")
+    kafka {
+      input.telemetry.topic = {{ env_name }}.telemetry.error
+      input.summary.topic = {{ env_name }}.telemetry.derived
+      telemetry.denorm.output.topic = {{ env_name }}.druid.events.error
+      summary.denorm.output.topic = {{ env_name }}.druid.events.summary
+      summary.unique.events.topic = {{ env_name }}.telemetry.derived.unique
+      output.failed.topic = {{ env_name }}.telemetry.failed
+      output.duplicate.topic = {{ env_name }}.telemetry.duplicate
+      groupId = {{ env_name }}-error-denorm-group
+    }
+    skip.events = ["INTERRUPT"]
+    permit.eid=["AUDIT"]
+    task {
+      window.count = {{ denorm_primary_window_count }}
+      window.shards = {{ denorm_primary_window_shards }}
+      consumer.parallelism = {{ error_denorm_consumer_parallelism }}
+      telemetry.downstream.operators.parallelism = {{ error_denorm_operators_parallelism }}
+      summary.downstream.operators.parallelism = {{ summary_denorm_operators_parallelism  }}
+    }
+    redis {
+      database {
+        duplicationstore.id = 9
+        key.expiry.seconds = {{ de_normalization_duplicationstore_key_expiry_seconds }} 
+      }
+    }
+    # redis-metadata
+    redis-meta {
+      database {
+        devicestore.id = 2
+        userstore.id = 12
+        contentstore.id = 5
+        dialcodestore.id = 6
+        key.expiry.seconds = {{ de_normalization_key_expiry_seconds }}
+      }
+      content.host = {{ redis_meta_content_host }}
+      device.host = {{ redis_meta_device_host }}
+      user.host = {{ redis_meta_user_host }}
+      dialcode.host = {{ redis_meta_dialcode_host }}
+      content.port = {{ redis_meta_content_port }}
+      device.port = {{ redis_meta_device_port }}
+      user.port = {{ redis_meta_user_port }}
+      dialcode.port = {{ redis_meta_dialcode_port }}
+    }
+    # config version
+    user_denorm_version = v2
+
+  flink-conf: |+
+    jobmanager.memory.flink.size: {{ flink_job_names['error-denormalization'].jobmanager_memory }}
+    taskmanager.memory.flink.size: {{ flink_job_names['error-denormalization'].taskmanager_memory }}
+    taskmanager.numberOfTaskSlots: {{ flink_job_names['error-denormalization'].taskslots }}
+    parallelism.default: 1
+    jobmanager.execution.failover-strategy: region
+    taskmanager.memory.network.fraction: 0.1
diff --git a/kubernetes/helm_charts/druid-cluster/.helmignore b/kubernetes/helm_charts/druid-cluster/.helmignore
new file mode 100644
index 0000000000..fbe01f88f2
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file
diff --git a/kubernetes/helm_charts/druid-cluster/Chart.yaml b/kubernetes/helm_charts/druid-cluster/Chart.yaml
new file mode 100644
index 0000000000..9588f92cc4
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/Chart.yaml
@@ -0,0 +1,18 @@
+name: druid-cluster
+version: 0.1.0
+appVersion: 0.21.1
+description: Helm chart to deploy a druid cluster on Kubernetes
+keywords:
+- druid
+- zookeeper
+maintainers:
+- name: druid-operator-maintainers
+sources:
+- https://github.com/druid-io/druid-operator
+engine: gotpl
+home: https://github.com/druid-io/druid-operator
+dependencies:
+- name: zookeeper
+  version: 7.4.0
+  repository: https://charts.bitnami.com/bitnami
+  condition: zookeeper.enabled
diff --git a/kubernetes/helm_charts/druid-cluster/requirements.lock b/kubernetes/helm_charts/druid-cluster/requirements.lock
new file mode 100644
index 0000000000..3ce72f9405
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/requirements.lock
@@ -0,0 +1,6 @@
+dependencies:
+- name: zookeeper
+  repository: https://charts.bitnami.com/bitnami
+  version: 7.4.0
+digest: sha256:ac1eacfdf8dca15edf525810420baab03ce1e0150c9d70305df0c56cdac19dce
+generated: "2021-08-25T22:12:41.072958472+05:30"
diff --git a/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl b/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl
new file mode 100644
index 0000000000..f05bab6764
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/templates/_helpers.tpl
@@ -0,0 +1,51 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "druid-cluster.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "name" -}}
+{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "druid-cluster.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "druid-cluster.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+App config names
+*/}}
+{{- define "druid-cluster.appConfigName" -}}
+{{- $.Release.Name -}}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "druid-cluster.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "druid-cluster.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml b/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml
new file mode 100644
index 0000000000..5fcb3e70e3
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/templates/druid_statefulset.yaml
@@ -0,0 +1,490 @@
+# This spec only works on a single node kubernetes cluster(e.g. typical k8s cluster setup for dev using kind/minikube or single node AWS EKS cluster etc)
+# as it uses local disk as "deep storage".
+#
+apiVersion: "druid.apache.org/v1alpha1"
+kind: "Druid"
+metadata:
+  name: "{{ .Values.druid_cluster_type }}-{{ .Values.druid_env }}"
+spec:
+  image: "{{ .Values.druid_image }}"
+  # Optionally specify image for all nodes. Can be specify on nodes also
+  # imagePullSecrets:
+  # - name: tutu
+  startScript: /druid.sh
+  podLabels:
+    environment: {{ .Values.druid_env }}
+    release: alpha
+  podAnnotations:
+    dummykey: dummyval
+  readinessProbe:
+    httpGet:
+      path: /status/health
+      port: 8088
+  securityContext:
+    fsGroup: 1000
+    runAsUser: 1000
+    runAsGroup: 1000
+  services:
+    - spec:
+        type: ClusterIP
+        clusterIP: None
+  commonConfigMountPath: "/opt/druid/conf/druid/cluster/_common"
+  # volumeMounts:
+  #   - mountPath: /druid/data
+  #     name: data-volume
+  # volumes:
+  #   - name: data-volume
+  #     emptyDir: {}
+  jvm.options: |-
+    -server
+    -XX:+UseG1GC
+    -XX:+ExitOnOutOfMemoryError
+    -XX:MaxDirectMemorySize=10240g
+    -Duser.timezone=UTC
+    -Dfile.encoding=UTF-8
+    -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+    -Djava.io.tmpdir={{ .Values.druid_default_tmp_dir }}
+  log4j.config: |-
+    <?xml version="1.0" encoding="UTF-8" ?>
+    <Configuration status="WARN">
+        <Appenders>
+            <Console name="Console" target="SYSTEM_OUT">
+                <PatternLayout pattern="%d{ISO8601} %p [%t] %c - %m%n"/>
+            </Console>
+        </Appenders>
+        <Loggers>
+            <Root level="info">
+                <AppenderRef ref="Console"/>
+            </Root>
+        </Loggers>
+    </Configuration>
+  common.runtime.properties: |
+    druid.extensions.loadList=[{{ .Values.druid_extensions_loadList }}]
+    # druid.extensions.directory=/opt/druid/extensions
+
+    # Logging
+    # Log all runtime properties on startup. Disable to avoid logging properties on startup:
+    druid.startup.logging.logProperties=true
+
+    # Zookeeper
+    # druid.zk.service.host=druid-cluster-zookeeper-headless.{{ .Values.druid_namespace }}.svc.cluster.local
+    druid.zk.service.host=druid-cluster-zookeeper-0.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181,druid-cluster-zookeeper-1.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181,druid-cluster-zookeeper-2.druid-cluster-zookeeper-headless.druid-raw.svc.cluster.local:2181
+
+    druid.zk.paths.base=/druid
+
+    # Metadata storage
+    # For PostgreSQL:
+    druid.metadata.storage.type={{ .Values.druid_metadata_storage_type }}
+    druid.metadata.storage.connector.connectURI={{ .Values.druid_metadata_storage_connector_connectURI }}
+    druid.metadata.storage.connector.user={{ .Values.druid_metadata_storage_connector_user }}
+    druid.metadata.storage.connector.password={{ .Values.druid_metadata_storage_connector_password }}
+
+    # Deep storage
+    #Using azure
+    druid.storage.type={{ .Values.druid_deepstorage_type }}
+    druid.azure.account={{ .Values.druid_azure_storage_account }}
+    druid.azure.key={{ .Values.druid_azure_storage_account_key }}
+    druid.azure.container={{ .Values.druid_azure_container }}
+
+    # Indexing service logs
+    # For local disk (only viable in a cluster if this is a network mount):
+    druid.indexer.logs.type={{ .Values.druid_indexer_logs_type }}
+    druid.indexer.logs.container={{ .Values.druid_indexer_logs_container }}
+    druid.indexer.logs.prefix= {{ .Values.druid_indexer_logs_prefix }}
+
+    # Service discovery
+    druid.selectors.indexing.serviceName={{ .Values.druid_selectors_indexing_serviceName }}
+    druid.selectors.coordinator.serviceName={{ .Values.druid_selectors_coordinator_serviceName }}
+
+    # Monitoring
+    # druid.monitoring.monitors=[{{ .Values.druid_monitoring_monitors }}]
+    # druid.emitter=composing
+    # druid.emitter.composing.emitters=[{{ .Values.druid_emitter_composing_emitters }}]
+    # druid.emitter.logging.logLevel={{ .Values.druid_emitter_logging_logLevel }}
+
+    {{- if .Values.druid_monitoring -}}
+    druid.emitter.graphite.port={{ .Values.druid_emitter_graphite_port }}
+    druid.emitter.graphite.hostname={{ .Values.druid_emitter_graphite_hostname }}
+    druid.emitter.graphite.protocol={{ .Values.druid_emitter_graphite_protocol }}
+    druid.emitter.graphite.eventConverter={{ .Values.druid_emitter_graphite_eventConverter }}
+    {{- end -}}
+
+    # Storage type of double columns
+    # ommiting this will lead to index double as float at the storage layer
+
+    druid.indexing.doubleStorage={{ .Values.druid_indexing_doubleStorage }}
+
+    #Writing query logs into file 
+    druid.request.logging.type={{ .Values.druid_request_logging_type }}
+    druid.request.logging.dir={{ .Values.druid_request_logging_dir }}
+
+    druid.javascript.enabled=true
+    druid.sql.enable={{ .Values.druid_sql_enable }}
+
+  env:
+    - name: POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: POD_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+
+  nodes:
+    brokers:
+      # Optionally specify for running broker as Deployment
+      kind: Deployment
+      nodeType: "broker"
+      # Optionally specify for broker nodes
+      # imagePullSecrets:
+      # - name: tutu
+      druid.port: {{ .Values.druid_broker_port }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/broker"
+      replicas: {{ .Values.druid_broker_replicas }}
+      services:
+        - spec:
+            type: NodePort
+            ports:
+            - name: druidbrokerport
+              port: 8082
+              targetPort: 8082
+              nodePort: 32000
+              protocol: TCP
+      runtime.properties: |
+        druid.service={{ .Values.druid_broker_service }}
+
+        # HTTP server threads
+        druid.broker.http.numConnections={{ .Values.druid_broker_http_numConnections }}
+        druid.server.http.numThreads={{ .Values.druid_broker_server_http_numThreads }}
+
+        # Processing threads and buffers
+        druid.processing.buffer.sizeBytes={{ .Values.druid_broker_processing_buffer_sizeBytes }}
+        druid.processing.numThreads={{ .Values.druid_broker_processing_numThreads }}
+
+        druid.javascript.enabled=true
+        druid.sql.enable={{ .Values.druid_sql_enable }}
+
+      extra.jvm.options: |-
+        -Xms{{ .Values.druid_broker_min_heap_size }}
+        -Xmx{{ .Values.druid_broker_max_heap_size }}
+        -XX:MaxDirectMemorySize={{ .Values.druid_broker_max_direct_size }}
+
+      # volumeMounts:
+      #   - mountPath: /druid/data
+      #     name: data-volume
+      # volumes:
+      #   - name: data-volume
+      #     emptyDir: {}
+
+      readinessProbe:
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_broker_port }}
+
+      hpAutoscaler:
+        maxReplicas: 10
+        minReplicas: 1
+        scaleTargetRef:
+           apiVersion: apps/v1
+           kind: StatefulSet
+           name: druid-tiny-cluster-brokers
+        metrics:
+         - type: Resource
+           resource:
+             name: cpu
+             target:
+               type: Utilization
+               averageUtilization: 50
+
+    coordinators:
+      # Optionally specify for running coordinator as Deployment
+      kind: Deployment
+      nodeType: "coordinator"
+      druid.port: {{ .Values.druid_coordinator_port }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord"
+      replicas: {{ .Values.druid_coordinator_replicas }}
+      runtime.properties: |
+        druid.service={{ .Values.druid_coordinator_service }}
+        druid.coordinator.startDelay={{ .Values.druid_coordinator_startDelay }}
+        druid.coordinator.period={{ .Values.druid_coordinator_period }}
+        druid.coordinator.balancer.strategy={{ .Values.druid_coordinator_balancer_strategy }}
+        druid.coordinator.asOverlord.enabled=false
+
+      extra.jvm.options: |-
+        -Xms{{ .Values.druid_coordinator_heap_size }}
+        -Xmx{{ .Values.druid_coordinator_heap_size }}
+
+      # volumeMounts:
+      #   - mountPath: /druid/data
+      #     name: data-volume
+      # volumes:
+      #   - name: data-volume
+      #     emptyDir: {}
+
+      readinessProbe:
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_coordinator_port }}
+
+    overlords:
+      # Optionally specify for running coordinator as Deployment
+      kind: Deployment
+      nodeType: "overlord"
+      druid.port: {{ .Values.druid_overlord_port }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/master/coordinator-overlord"
+      replicas: {{ .Values.druid_overlord_replicas }}
+      runtime.properties: |
+        druid.service={{ .Values.druid_overlord_service }}
+
+        druid.indexer.queue.startDelay={{ .Values.druid_indexer_queue_startDelay }}
+
+        druid.indexer.runner.type={{ .Values.druid_indexer_runner_type }}
+        druid.indexer.storage.type={{ .Values.druid_indexer_storage_type }}
+
+        # Additional parameters for minor compaction
+        druid.indexer.tasklock.forceTimeChunkLock={{ .Values.druid_indexer_tasklock_forceTimeChunkLock }}
+
+      extra.jvm.options: |-
+        -Xms{{ .Values.druid_overlord_heap_size }}
+        -Xmx{{ .Values.druid_overlord_heap_size }}
+
+      # volumeMounts:
+      #   - mountPath: /druid/data
+      #     name: data-volume
+      # volumes:
+      #   - name: data-volume
+      #     emptyDir: {}
+
+      readinessProbe:
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_overlord_port }}
+
+    historicals:
+      nodeType: "historical"
+      druid.port: {{ .Values.druid_historical_port }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/historical"
+      replicas: {{ .Values.druid_historical_replicas }}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: "nodeSpecUniqueStr"
+                  operator: In
+                  values:
+                  - {{ .Values.druid_namespace }}-{{ .Values.druid_env }}-historicals
+                  - {{ .Values.druid_namespace }}-{{ .Values.druid_env }}-middlemanagers
+            topologyKey: "kubernetes.io/hostname"
+      readinessProbe:
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_historical_port }}
+      runtime.properties: |
+        druid.service={{ .Values.druid_historical_service }}
+
+        # HTTP server threads
+        druid.server.http.numThreads={{ .Values.druid_historical_server_http_numThreads }}
+
+        # Processing threads and buffers
+        druid.processing.buffer.sizeBytes={{ .Values.druid_historical_processing_buffer_sizeBytes }}
+        druid.processing.numThreads={{ .Values.druid_historical_processing_numThreads }}
+        druid.processing.numMergeBuffers={{ .Values.druid_historical_processing_numMergeBuffers }}
+
+        # Segmentstorage
+        druid.segmentCache.locations=[{{ .Values.druid_segmentCache_locations }}]
+
+        druid.segmentCache.numLoadingThreads={{ .Values.druid_segmentCache_numLoadingThreads }}
+
+        # Caching
+        druid.historical.cache.useCache={{ .Values.druid_historical_cache_useCache }} 
+        druid.historical.cache.populateCache={{ .Values.druid_historical_cache_populateCache }}
+        druid.historical.cache.unCacheable=[{{ .Values.druid_historical_cache_unCacheable }}]
+        druid.cache.type={{ .Values.druid_cache_type }}
+        druid.cache.sizeInBytes={{ .Values.druid_historical_cache_size }}
+        # druid.cache.expireAfter={{ .Values.druid_historical_cache_expiry }}
+
+      extra.jvm.options: |-
+        -Xms{{ .Values.druid_historical_min_heap_size }}
+        -Xmx{{ .Values.druid_historical_max_heap_size }}
+        -XX:MaxDirectMemorySize={{ .Values.druid_historical_max_direct_size }}
+
+      securityContext:
+        fsGroup: 0
+        runAsUser: 0
+        runAsGroup: 0
+
+      volumeMounts:
+        - mountPath: /druid/data
+          name: historical-volume
+      volumeClaimTemplates:
+      - metadata:
+          name: historical-volume
+        spec:
+          storageClassName: "{{- .Values.storageClass }}"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: {{ .Values.druid_historical_persistent_volume_size }}
+
+      # volumes:
+      #   - name: historical-volume
+      #     persistentVolumeClaim:
+      #       claimName: historical-pv-claim
+      #   # - name: data-volume
+      #   #   emptyDir: {}
+      # volumeMounts:
+      #   - mountPath: "/var/segments"
+      #     name: historical-volume
+      #   # - mountPath: /druid/data
+      #   #   name: data-volume
+      
+    middlemanagers:
+      nodeType: "middleManager"
+      druid.port: {{ .Values.druid_middlemanager_port }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/data/middleManager"
+      replicas: {{ .Values.druid_middlemanager_replicas }}
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: "nodeSpecUniqueStr"
+                  operator: In
+                  values:
+                  - {{ .Values.druid_namespace }}-{{ .Values.druid_env }}-historicals
+                  - {{ .Values.druid_namespace }}-{{ .Values.druid_env }}-middlemanagers
+            topologyKey: "kubernetes.io/hostname"
+      runtime.properties: |
+        druid.service={{ .Values.druid_middlemanager_service }}
+
+        # Number of tasks per middleManager
+        druid.worker.capacity={{ .Values.druid_worker_capacity }}
+
+        # Task launch parameters
+        druid.indexer.runner.javaOpts={{ .Values.druid_indexer_runner_javaOpts }}
+        druid.indexer.task.baseTaskDir={{ .Values.druid_indexer_task_baseTaskDir }}
+
+        # Peon properties
+        druid.indexer.fork.property.druid.processing.buffer.sizeBytes={{ .Values.druid_indexer_fork_property_druid_processing_buffer_sizeBytes }}
+        druid.indexer.fork.property.druid.processing.numThreads={{ .Values.druid_indexer_fork_property_druid_processing_numThreads }}
+        druid.indexer.fork.property.druid.server.http.numThreads={{ .Values.druid_indexer_fork_property_druid_server_http_numThreads }}
+
+        #Additional Parameters
+        druid.indexer.task.restoreTasksOnRestart={{ .Values.druid_indexer_task_restoreTasksOnRestart }}
+        druid.indexer.task.defaultHadoopCoordinates=[\"org.apache.hadoop:hadoop-client:2.8.3\"]
+
+      extra.jvm.options: |-
+        -Xmx{{ .Values.druid_middlemanager_heap_size }}
+        -Xms{{ .Values.druid_middlemanager_heap_size }}
+
+      # services:
+      # - spec:
+      #     clusterIP: None
+      #     ports:
+      #     - name: middlemanager-port
+      #       port: {{ .Values.druid_middlemanager_port }}
+      #       targetPort: {{ .Values.druid_middlemanager_port }}
+      #     type: ClusterIP
+
+      readinessProbe:
+        initialDelaySeconds: 30
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_middlemanager_port }}
+
+      securityContext:
+        fsGroup: 0
+        runAsUser: 0
+        runAsGroup: 0
+
+      volumeMounts:
+        - mountPath: /druid/data
+          name: middlemanager-volume
+      volumeClaimTemplates:
+      - metadata:
+          name: middlemanager-volume
+        spec:
+          storageClassName: "{{- .Values.storageClass }}"
+          accessModes:
+            - ReadWriteOnce
+          resources:
+            requests:
+              storage: {{ .Values.druid_middlemanager_persistent_volume_size }}
+
+      # volumeMounts:
+      #   - mountPath: /var/task
+      #     name: middlemanager-volume
+      #   # - mountPath: /druid/data
+      #   #   name: data-volume
+      # volumes:
+      #   - name: middlemanager-volume
+      #     persistentVolumeClaim:
+      #       claimName: middlemanager-pv-claim
+      #   # - name: data-volume
+      #   #   emptyDir: {}
+          
+    routers:
+      nodeType: "router"
+      druid.port: {{ .Values.druid_router_plaintextPort }}
+      nodeConfigMountPath: "/opt/druid/conf/druid/cluster/query/router"
+      replicas: {{ .Values.druid_router_replicas }}
+      services:
+        - spec:
+            type: ClusterIP
+            ports:
+            - name: druidrouterport
+              port: 80
+              targetPort: 8888
+              protocol: TCP
+      ingressAnnotations:
+        name: router-ingress
+        nginx.ingress.kubernetes.io/rewrite-target: /$1
+      ingress:
+        ingressClassName: nginx
+        rules:
+          - host: "*.nip.io"
+            http:
+              paths:
+                - path: /
+                  pathType: Prefix
+                  backend:
+                    service:
+                      name: druid-{{ .Values.druid_cluster_type }}-{{ .Values.druid_env }}-routers
+                      port:
+                        name: druidrouterport
+
+      runtime.properties: |
+        druid.service={{ .Values.druid_router_service }}
+        # druid.plaintextPort={{ .Values.druid_router_plaintextPort }}
+
+        # HTTP proxy
+        druid.router.http.numConnections={{ .Values.druid_router_http_numConnections }}
+        druid.router.http.readTimeout={{ .Values.druid_router_http_readTimeout }}
+        druid.router.http.numMaxThreads={{ .Values.druid_router_http_numMaxThreads }}
+        druid.server.http.numThreads={{ .Values.druid_router_server_http_numThreads }}
+
+        # Service discovery
+        druid.router.defaultBrokerServiceName={{ .Values.druid_broker_service }}
+        druid.router.coordinatorServiceName={{ .Values.druid_coordinator_service }}
+
+        # Management proxy to coordinator / overlord: required for unified web console.
+        druid.router.managementProxy.enabled={{ .Values.druid_router_managementProxy_enabled }}
+
+      extra.jvm.options: |-
+        -Xms{{ .Values.druid_router_heap_size }}
+        -Xmx{{ .Values.druid_router_heap_size }}
+
+      # volumeMounts:
+      #   - mountPath: /druid/data
+      #     name: data-volume
+      # volumes:
+      #   - name: data-volume
+      #     emptyDir: {}
+
+      readinessProbe:
+        httpGet:
+          path: /status/health
+          port: {{ .Values.druid_router_plaintextPort }}
diff --git a/kubernetes/helm_charts/druid-cluster/values.j2 b/kubernetes/helm_charts/druid-cluster/values.j2
new file mode 100644
index 0000000000..169ef07b8e
--- /dev/null
+++ b/kubernetes/helm_charts/druid-cluster/values.j2
@@ -0,0 +1,274 @@
+druid_env: "{{ env }}"
+druid_cluster_type: "{{ cluster_type }}"
+druid_image: "{{ druid_image }}"
+druid_namespace: "{{ druid_namespace }}"
+dp_vault_pgdb_admin_password: postgres
+druid_monitoring: {{ druid_monitoring }}
+
+storageClass: {{ persistent_storage_class }}
+
+######################### Druid common variables ########################
+
+druid_directory: "/opt/druid"
+druid_extensions_loadList: '{{ druid_extensions_loadList }}'
+
+# Logging
+# Log all runtime properties on startup. Disable to avoid logging properties on startup:
+druid.startup.logging.logProperties: true
+
+# Druid Metadata Store
+druid_metadata_storage_type: "postgresql"
+druid_metadata_storage_connector_connectURI: "jdbc:postgresql://{{ druid_configs[cluster_type].druid_postgres_host }}:{{ druid_configs[cluster_type].druid_postgres_port }}/{{ druid_configs[cluster_type].druid_postgres_db }}"
+druid_metadata_storage_connector_user: {{ druid_configs[cluster_type].druid_postgres_user }}
+druid_metadata_storage_connector_password: {{ druid_configs[cluster_type].druid_postgres_pass }}
+
+# Druid Storage Type
+druid_deepstorage_type: azure
+druid_azure_storage_account: "{{ sunbird_private_storage_account_name }}"
+druid_azure_storage_account_key: "{{ sunbird_private_storage_account_key }}"
+druid_azure_container: "{{ druid_configs[cluster_type].azure_container }}"
+
+# Indexing service logs
+# For local disk (only viable in a cluster_type if this is a network mount):
+druid_indexer_logs_type: "{{ druid_indexing_logs_type }}"
+druid_indexer_logs_container: "{{ druid_configs[cluster_type].druid_log_azure_container }}"
+druid_indexer_logs_prefix: "{{ druid_configs[cluster_type].druid_log_azure_folder }}"
+
+# Service discovery
+druid_selectors_indexing_serviceName: {{ druid_overlord_service }}
+druid_selectors_coordinator_serviceName: {{ druid_coordinator_service }}
+
+# Monitoring
+druid_monitoring_monitors: '{{ druid_common_monitors }}'
+druid_emitter: composing
+druid_emitter_composing_emitters: '{{ druid_common_emitters }}'
+druid_emitter_logging_logLevel: {{ druid_emitter_logging_logLevel }}
+
+{% if druid_monitoring %}
+druid_emitter_graphite_port: {{ druid_graphite_port | d(9109)}}
+druid_emitter_graphite_hostname: {{ druid_graphite_host | d("localhost") }}
+druid_emitter_graphite_protocol: plaintext
+druid_emitter_graphite_eventConverter: '{"type":"whiteList", "namespacePrefix": "{{ druid_graphite_prefix }}", "ignoreHostname":false, "ignoreServiceName":false ,"mapPath":"{{ druid_whitelist_filepath }}"}'
+{% endif %}
+
+# Storage type of double columns
+# ommiting this will lead to index double as float at the storage layer
+druid_indexing_doubleStorage: double
+
+druid_default_tmp_dir: {{druid_default_tmp_dir}}
+
+#Writing query logs into file 
+druid_request_logging_type: {{ druid_request_logging_type }}
+druid_request_logging_dir: "{{ druid_log_dir }}"
+
+druid_javascript_enabled: true
+druid_sql_enable: {{ enable_druid_sql | lower }}
+
+####################### Druid Broker Variables ##########################
+
+druid_broker_service: {{ druid_broker_service }}
+druid_broker_port: {{ druid_broker_port }}
+druid_broker_min_heap_size: {{ druid_configs[cluster_type].druid_broker_min_heap_size }}
+druid_broker_max_heap_size: {{ druid_configs[cluster_type].druid_broker_max_heap_size }}
+druid_broker_max_direct_size: {{ druid_configs[cluster_type].druid_broker_max_direct_size }}
+
+# HTTP server threads
+druid_broker_http_numConnections: {{ druid_configs[cluster_type].druid_broker_http_numConnections }}
+druid_broker_server_http_numThreads: {{ druid_configs[cluster_type].druid_broker_server_http_numThread }}
+
+# Processing threads and buffers
+druid_broker_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_broker_processing_bufferBytes }}
+druid_broker_processing_numThreads: {{ druid_configs[cluster_type].druid_broker_processing_threads }}
+
+druid_javascript.enabled: true
+druid_sql_enable: {{ enable_druid_sql | lower }}
+
+druid_broker_replicas: {{ druid_configs[cluster_type].druid_broker_replicas }}
+
+##################### Druid Coordinator Variables #######################
+
+druid_coordinator_service: {{ druid_coordinator_service }}
+druid_coordinator_port: {{ druid_coordinator_port }}
+druid_coordinator_heap_size: {{ druid_configs[cluster_type].druid_coordinator_heap_size }}
+
+druid_coordinator_startDelay: {{ druid_configs[cluster_type].druid_coordinator_startDelay }}
+druid_coordinator_period: {{ druid_configs[cluster_type].druid_coordinator_period }}
+druid_coordinator_balancer_strategy: {{ druid_configs[cluster_type].druid_coordinator_balance_strategy }}
+
+druid_coordinator_replicas: {{ druid_configs[cluster_type].druid_coordinator_replicas }}
+
+####################### Druid Overlord Variables ########################
+
+druid_overlord_service: {{ druid_overlord_service }}
+druid_overlord_port: {{ druid_overlord_port }}
+druid_overlord_heap_size: {{ druid_configs[cluster_type].druid_overlord_heap_size }}
+
+druid_indexer_queue_startDelay: {{ druid_configs[cluster_type].druid_indexing_queue_startDelay }}
+
+druid_indexer_runner_type: remote
+druid_indexer_storage_type: {{ druid_indexing_storage_type }}
+
+# Additional parameters for minor compaction
+druid_indexer_tasklock_forceTimeChunkLock: false
+
+druid_overlord_replicas: {{ druid_configs[cluster_type].druid_overlord_replicas }}
+
+###################### Druid Historical Variables #######################
+
+druid_historical_service: {{ druid_historical_service }}
+druid_historical_port: {{ druid_historical_port }}
+druid_historical_min_heap_size: {{ druid_configs[cluster_type].druid_historical_min_heap_size }}
+druid_historical_max_heap_size: {{ druid_configs[cluster_type].druid_historical_max_heap_size }}
+druid_historical_max_direct_size: {{ druid_configs[cluster_type].druid_historical_max_direct_size }}
+
+# HTTP server threads
+druid_historical_server_http_numThreads: {{ druid_configs[cluster_type].druid_historical_server_http_numThread }}
+
+# Processing threads and buffers
+druid_historical_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_historical_processing_bufferBytes }}
+druid_historical_processing_numThreads: {{ druid_configs[cluster_type].druid_historical_processing_threads }}
+druid_historical_processing_numMergeBuffers: {{ druid_configs[cluster_type].druid_historical_processing_num_merge_buffers }}
+
+druid_query_ondiskstorage_enabled: {{ druid_configs[cluster_type].druid_query_ondiskstorage_enabled }}
+druid_query_groupBy_maxMergingDictionarySize: {{ druid_configs[cluster_type].druid_historical_maxMergingDictionarySize }}
+druid_query_groupBy_maxOnDiskStorage: {{ druid_configs[cluster_type].druid_query_groupBy_maxOnDiskStorage }}
+
+# Segmentstorage
+druid_segmentCache_locations: '{% for index in range(5) %}{"path": "{{ druid_configs[cluster_type].druid_historical_segmentcache_path }}{{index}}", "maxSize": {{ druid_configs[cluster_type].druid_historical_segmentcache_size }}}{% if not loop.last %}, {% endif %}
+{% endfor %}'
+
+druid_historical_persistent_volume_size: {{ druid_configs[cluster_type].druid_historical_persistent_volume_size }}
+
+druid_segmentCache_numLoadingThreads: {{ druid_configs[cluster_type].druid_historical_segmentcache_numloadingthreads }}
+
+# Caching
+druid_historical_cache_useCache: {{ druid_configs[cluster_type].druid_historical_enable_cache }} 
+druid_historical_cache_populateCache: true
+druid_historical_cache_unCacheable: '{{ druid_historical_cache_unCacheable }}'
+druid_cache_type: caffeine
+druid_historical_cache_size: {{ druid_historical_cache_size }}
+druid_historical_cache_expiry: {{ druid_historical_cache_expiry }}
+
+druid_historical_replicas: {{ druid_configs[cluster_type].druid_historical_replicas }}
+
+#################### Druid Middlemanager Variables ######################
+
+druid_middlemanager_service: {{ druid_middlemanager_service }}
+druid_middlemanager_port: {{ druid_middlemanager_port }}
+druid_middlemanager_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_heap_size }}
+
+druid_middlemanager_persistent_volume_size: {{ druid_configs[cluster_type].druid_middlemanager_persistent_volume_size }}
+
+# Number of tasks per middleManager
+druid_worker_capacity: {{ druid_configs[cluster_type].druid_middlemanager_worker_cap }}
+druid_middlemanager_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_heap_size }}
+druid_middlemanager_peon_heap_size: {{ druid_configs[cluster_type].druid_middlemanager_peon_heap_size}}
+
+# Task launch parameters
+druid_indexer_runner_javaOpts: "{{ druid_configs[cluster_type].druid_mm_java_opts_array }}"
+druid_indexer_task_baseTaskDir: "{{ druid_indexing_task_basedir }}"
+
+# Peon properties
+druid_indexer_fork_property_druid_processing_buffer_sizeBytes: {{ druid_configs[cluster_type].druid_middlemanager_peon_processing_bufferBytes }}
+druid_indexer_fork_property_druid_processing_numThreads: {{ druid_configs[cluster_type].druid_middlemanager_peon_processing_threads }}
+druid_indexer_fork_property_druid_server_http_numThreads: {{ druid_configs[cluster_type].druid_middlemanager_peon_server_http_numThread }}
+
+#Additional Parameters
+druid_indexer_task_restoreTasksOnRestart: true
+
+druid_middlemanager_replicas: {{ druid_configs[cluster_type].druid_middlemanager_replicas }}
+
+######################## Druid Router Variables #########################
+
+druid_router_service: {{ druid_router_service }}
+druid_router_plaintextPort: {{ druid_router_plaintextPort }}
+druid_router_heap_size: {{ druid_configs[cluster_type].druid_router_heap_size }}
+
+# HTTP proxy
+druid_router_http_numConnections: {{ druid_configs[cluster_type].druid_router_http_numConnections }}
+druid_router_http_readTimeout: {{ druid_configs[cluster_type].druid_router_http_readTimeout }}
+druid_router_http_numMaxThreads: {{ druid_configs[cluster_type].druid_router_http_numMaxThreads }}
+druid_router_server_http_numThreads: {{ druid_configs[cluster_type].druid_router_server_http_numThreads }}
+
+# Service discovery
+druid_router_defaultBrokerServiceName: {{ druid_broker_service }}
+druid_router_coordinatorServiceName: {{ druid_coordinator_service }}
+
+# Management proxy to coordinator / overlord: required for unified web console.
+druid_router_managementProxy_enabled: {{ druid_configs[cluster_type].druid_router_managementProxy_enabled }}
+
+druid_router_replicas: {{ druid_configs[cluster_type].druid_router_replicas }}
+
+#################### Zookeeper Variables ######################
+
+zookeeper:
+  image:
+    registry: docker.io
+    repository: bitnami/zookeeper
+    tag: 3.6-debian-10
+    pullPolicy: Always
+
+  heapSize: 256
+  replicaCount: 3
+
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 250m
+
+  tickTime: 2000
+  initLimit: 10
+  syncLimit: 5
+  preAllocSize: 65536
+  snapCount: 100000
+  maxClientCnxns: 60
+  fourlwCommandsWhitelist: srvr, mntr, ruok
+  listenOnAllIPs: false
+  allowAnonymousLogin: true
+  autopurge:
+    snapRetainCount: 3
+    purgeInterval: 0
+  maxSessionTimeout: 40000
+
+  allowAnonymousLogin: true
+
+  minServerId: 1
+
+  securityContext:
+    enabled: true
+    fsGroup: 1001
+    runAsUser: 1001
+
+  livenessProbe:
+    enabled: true
+    initialDelaySeconds: 30
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 6
+    successThreshold: 1
+    probeCommandTimeout: 2
+
+  readinessProbe:
+    enabled: true
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    failureThreshold: 6
+    successThreshold: 1
+    probeCommandTimeout: 2
+
+  service:
+    type: ClusterIP
+    # loadBalancerIP: ""
+    port: 2181
+    followerPort: 2888
+    electionPort: 3888
+    nodePorts:
+      client: ""
+      clientTls: ""
+    publishNotReadyAddresses: true
+    tlsClientPort: 3181
+    disableBaseClientPort: false
+    annotations: {}
+    headless:
+      annotations: {}
diff --git a/kubernetes/helm_charts/druid-operator/Chart.yaml b/kubernetes/helm_charts/druid-operator/Chart.yaml
new file mode 100644
index 0000000000..6588f1e73d
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/Chart.yaml
@@ -0,0 +1,23 @@
+apiVersion: v2
+name: druid-operator
+description: Druid Kubernetes Operator
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 0.1.1
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+appVersion: 0.0.7
diff --git a/kubernetes/helm_charts/druid-operator/templates/NOTES.txt b/kubernetes/helm_charts/druid-operator/templates/NOTES.txt
new file mode 100644
index 0000000000..88a89946c8
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/NOTES.txt
@@ -0,0 +1 @@
+Refer to https://github.com/druid-io/druid-operator/blob/master/docs/README.md to get started.
diff --git a/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl b/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl
new file mode 100644
index 0000000000..628079bfe6
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/_helpers.tpl
@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "druid-operator.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "druid-operator.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "druid-operator.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "druid-operator.labels" -}}
+helm.sh/chart: {{ include "druid-operator.chart" . }}
+{{ include "druid-operator.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "druid-operator.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "druid-operator.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "druid-operator.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "druid-operator.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml b/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml
new file mode 100644
index 0000000000..ed176311c0
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/crds/druid.apache.org_druids.yaml
@@ -0,0 +1,4467 @@
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0-beta.0
+  creationTimestamp: null
+  name: druids.druid.apache.org
+spec:
+  group: druid.apache.org
+  names:
+    kind: Druid
+    listKind: DruidList
+    plural: druids
+    singular: druid
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        properties:
+          apiVersion:
+            type: string
+          kind:
+            type: string
+          metadata:
+            type: object
+          spec:
+            properties:
+              affinity:
+                properties:
+                  nodeAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            preference:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - preference
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        properties:
+                          nodeSelectorTerms:
+                            items:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchFields:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                              type: object
+                            type: array
+                        required:
+                        - nodeSelectorTerms
+                        type: object
+                    type: object
+                  podAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                  podAntiAffinity:
+                    properties:
+                      preferredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            podAffinityTerm:
+                              properties:
+                                labelSelector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                namespaces:
+                                  items:
+                                    type: string
+                                  type: array
+                                topologyKey:
+                                  type: string
+                              required:
+                              - topologyKey
+                              type: object
+                            weight:
+                              format: int32
+                              type: integer
+                          required:
+                          - podAffinityTerm
+                          - weight
+                          type: object
+                        type: array
+                      requiredDuringSchedulingIgnoredDuringExecution:
+                        items:
+                          properties:
+                            labelSelector:
+                              properties:
+                                matchExpressions:
+                                  items:
+                                    properties:
+                                      key:
+                                        type: string
+                                      operator:
+                                        type: string
+                                      values:
+                                        items:
+                                          type: string
+                                        type: array
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                              type: object
+                            namespaces:
+                              items:
+                                type: string
+                              type: array
+                            topologyKey:
+                              type: string
+                          required:
+                          - topologyKey
+                          type: object
+                        type: array
+                    type: object
+                type: object
+              common.runtime.properties:
+                type: string
+              commonConfigMountPath:
+                type: string
+              containerSecurityContext:
+                properties:
+                  allowPrivilegeEscalation:
+                    type: boolean
+                  capabilities:
+                    properties:
+                      add:
+                        items:
+                          type: string
+                        type: array
+                      drop:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  privileged:
+                    type: boolean
+                  procMount:
+                    type: string
+                  readOnlyRootFilesystem:
+                    type: boolean
+                  runAsGroup:
+                    format: int64
+                    type: integer
+                  runAsNonRoot:
+                    type: boolean
+                  runAsUser:
+                    format: int64
+                    type: integer
+                  seLinuxOptions:
+                    properties:
+                      level:
+                        type: string
+                      role:
+                        type: string
+                      type:
+                        type: string
+                      user:
+                        type: string
+                    type: object
+                  seccompProfile:
+                    properties:
+                      localhostProfile:
+                        type: string
+                      type:
+                        type: string
+                    required:
+                    - type
+                    type: object
+                  windowsOptions:
+                    properties:
+                      gmsaCredentialSpec:
+                        type: string
+                      gmsaCredentialSpecName:
+                        type: string
+                      runAsUserName:
+                        type: string
+                    type: object
+                type: object
+              deepStorage:
+                properties:
+                  spec:
+                    format: byte
+                    type: string
+                  type:
+                    type: string
+                required:
+                - spec
+                - type
+                type: object
+              deleteOrphanPvc:
+                type: boolean
+              disablePVCDeletionFinalizer:
+                type: boolean
+              env:
+                items:
+                  properties:
+                    name:
+                      type: string
+                    value:
+                      type: string
+                    valueFrom:
+                      properties:
+                        configMapKeyRef:
+                          properties:
+                            key:
+                              type: string
+                            name:
+                              type: string
+                            optional:
+                              type: boolean
+                          required:
+                          - key
+                          type: object
+                        fieldRef:
+                          properties:
+                            apiVersion:
+                              type: string
+                            fieldPath:
+                              type: string
+                          required:
+                          - fieldPath
+                          type: object
+                        resourceFieldRef:
+                          properties:
+                            containerName:
+                              type: string
+                            divisor:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                              x-kubernetes-int-or-string: true
+                            resource:
+                              type: string
+                          required:
+                          - resource
+                          type: object
+                        secretKeyRef:
+                          properties:
+                            key:
+                              type: string
+                            name:
+                              type: string
+                            optional:
+                              type: boolean
+                          required:
+                          - key
+                          type: object
+                      type: object
+                  required:
+                  - name
+                  type: object
+                type: array
+              envFrom:
+                items:
+                  properties:
+                    configMapRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                    prefix:
+                      type: string
+                    secretRef:
+                      properties:
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                  type: object
+                type: array
+              forceDeleteStsPodOnError:
+                type: boolean
+              ignored:
+                type: boolean
+              image:
+                type: string
+              imagePullPolicy:
+                type: string
+              imagePullSecrets:
+                items:
+                  properties:
+                    name:
+                      type: string
+                  type: object
+                type: array
+              jvm.options:
+                type: string
+              livenessProbe:
+                properties:
+                  exec:
+                    properties:
+                      command:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  failureThreshold:
+                    format: int32
+                    type: integer
+                  httpGet:
+                    properties:
+                      host:
+                        type: string
+                      httpHeaders:
+                        items:
+                          properties:
+                            name:
+                              type: string
+                            value:
+                              type: string
+                          required:
+                          - name
+                          - value
+                          type: object
+                        type: array
+                      path:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                      scheme:
+                        type: string
+                    required:
+                    - port
+                    type: object
+                  initialDelaySeconds:
+                    format: int32
+                    type: integer
+                  periodSeconds:
+                    format: int32
+                    type: integer
+                  successThreshold:
+                    format: int32
+                    type: integer
+                  tcpSocket:
+                    properties:
+                      host:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    required:
+                    - port
+                    type: object
+                  timeoutSeconds:
+                    format: int32
+                    type: integer
+                type: object
+              log4j.config:
+                type: string
+              metadataStore:
+                properties:
+                  spec:
+                    format: byte
+                    type: string
+                  type:
+                    type: string
+                required:
+                - spec
+                - type
+                type: object
+              nodeSelector:
+                additionalProperties:
+                  type: string
+                type: object
+              nodes:
+                additionalProperties:
+                  properties:
+                    affinity:
+                      properties:
+                        nodeAffinity:
+                          properties:
+                            preferredDuringSchedulingIgnoredDuringExecution:
+                              items:
+                                properties:
+                                  preference:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                      matchFields:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                    type: object
+                                  weight:
+                                    format: int32
+                                    type: integer
+                                required:
+                                - preference
+                                - weight
+                                type: object
+                              type: array
+                            requiredDuringSchedulingIgnoredDuringExecution:
+                              properties:
+                                nodeSelectorTerms:
+                                  items:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                      matchFields:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                    type: object
+                                  type: array
+                              required:
+                              - nodeSelectorTerms
+                              type: object
+                          type: object
+                        podAffinity:
+                          properties:
+                            preferredDuringSchedulingIgnoredDuringExecution:
+                              items:
+                                properties:
+                                  podAffinityTerm:
+                                    properties:
+                                      labelSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                      namespaces:
+                                        items:
+                                          type: string
+                                        type: array
+                                      topologyKey:
+                                        type: string
+                                    required:
+                                    - topologyKey
+                                    type: object
+                                  weight:
+                                    format: int32
+                                    type: integer
+                                required:
+                                - podAffinityTerm
+                                - weight
+                                type: object
+                              type: array
+                            requiredDuringSchedulingIgnoredDuringExecution:
+                              items:
+                                properties:
+                                  labelSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                  namespaces:
+                                    items:
+                                      type: string
+                                    type: array
+                                  topologyKey:
+                                    type: string
+                                required:
+                                - topologyKey
+                                type: object
+                              type: array
+                          type: object
+                        podAntiAffinity:
+                          properties:
+                            preferredDuringSchedulingIgnoredDuringExecution:
+                              items:
+                                properties:
+                                  podAffinityTerm:
+                                    properties:
+                                      labelSelector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                      namespaces:
+                                        items:
+                                          type: string
+                                        type: array
+                                      topologyKey:
+                                        type: string
+                                    required:
+                                    - topologyKey
+                                    type: object
+                                  weight:
+                                    format: int32
+                                    type: integer
+                                required:
+                                - podAffinityTerm
+                                - weight
+                                type: object
+                              type: array
+                            requiredDuringSchedulingIgnoredDuringExecution:
+                              items:
+                                properties:
+                                  labelSelector:
+                                    properties:
+                                      matchExpressions:
+                                        items:
+                                          properties:
+                                            key:
+                                              type: string
+                                            operator:
+                                              type: string
+                                            values:
+                                              items:
+                                                type: string
+                                              type: array
+                                          required:
+                                          - key
+                                          - operator
+                                          type: object
+                                        type: array
+                                      matchLabels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                    type: object
+                                  namespaces:
+                                    items:
+                                      type: string
+                                    type: array
+                                  topologyKey:
+                                    type: string
+                                required:
+                                - topologyKey
+                                type: object
+                              type: array
+                          type: object
+                      type: object
+                    containerSecurityContext:
+                      properties:
+                        allowPrivilegeEscalation:
+                          type: boolean
+                        capabilities:
+                          properties:
+                            add:
+                              items:
+                                type: string
+                              type: array
+                            drop:
+                              items:
+                                type: string
+                              type: array
+                          type: object
+                        privileged:
+                          type: boolean
+                        procMount:
+                          type: string
+                        readOnlyRootFilesystem:
+                          type: boolean
+                        runAsGroup:
+                          format: int64
+                          type: integer
+                        runAsNonRoot:
+                          type: boolean
+                        runAsUser:
+                          format: int64
+                          type: integer
+                        seLinuxOptions:
+                          properties:
+                            level:
+                              type: string
+                            role:
+                              type: string
+                            type:
+                              type: string
+                            user:
+                              type: string
+                          type: object
+                        seccompProfile:
+                          properties:
+                            localhostProfile:
+                              type: string
+                            type:
+                              type: string
+                          required:
+                          - type
+                          type: object
+                        windowsOptions:
+                          properties:
+                            gmsaCredentialSpec:
+                              type: string
+                            gmsaCredentialSpecName:
+                              type: string
+                            runAsUserName:
+                              type: string
+                          type: object
+                      type: object
+                    druid.port:
+                      format: int32
+                      type: integer
+                    env:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                          value:
+                            type: string
+                          valueFrom:
+                            properties:
+                              configMapKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                              fieldRef:
+                                properties:
+                                  apiVersion:
+                                    type: string
+                                  fieldPath:
+                                    type: string
+                                required:
+                                - fieldPath
+                                type: object
+                              resourceFieldRef:
+                                properties:
+                                  containerName:
+                                    type: string
+                                  divisor:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                    x-kubernetes-int-or-string: true
+                                  resource:
+                                    type: string
+                                required:
+                                - resource
+                                type: object
+                              secretKeyRef:
+                                properties:
+                                  key:
+                                    type: string
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                required:
+                                - key
+                                type: object
+                            type: object
+                        required:
+                        - name
+                        type: object
+                      type: array
+                    envFrom:
+                      items:
+                        properties:
+                          configMapRef:
+                            properties:
+                              name:
+                                type: string
+                              optional:
+                                type: boolean
+                            type: object
+                          prefix:
+                            type: string
+                          secretRef:
+                            properties:
+                              name:
+                                type: string
+                              optional:
+                                type: boolean
+                            type: object
+                        type: object
+                      type: array
+                    extra.jvm.options:
+                      type: string
+                    hpAutoscaler:
+                      properties:
+                        behavior:
+                          properties:
+                            scaleDown:
+                              properties:
+                                policies:
+                                  items:
+                                    properties:
+                                      periodSeconds:
+                                        format: int32
+                                        type: integer
+                                      type:
+                                        type: string
+                                      value:
+                                        format: int32
+                                        type: integer
+                                    required:
+                                    - periodSeconds
+                                    - type
+                                    - value
+                                    type: object
+                                  type: array
+                                selectPolicy:
+                                  type: string
+                                stabilizationWindowSeconds:
+                                  format: int32
+                                  type: integer
+                              type: object
+                            scaleUp:
+                              properties:
+                                policies:
+                                  items:
+                                    properties:
+                                      periodSeconds:
+                                        format: int32
+                                        type: integer
+                                      type:
+                                        type: string
+                                      value:
+                                        format: int32
+                                        type: integer
+                                    required:
+                                    - periodSeconds
+                                    - type
+                                    - value
+                                    type: object
+                                  type: array
+                                selectPolicy:
+                                  type: string
+                                stabilizationWindowSeconds:
+                                  format: int32
+                                  type: integer
+                              type: object
+                          type: object
+                        maxReplicas:
+                          format: int32
+                          type: integer
+                        metrics:
+                          items:
+                            properties:
+                              containerResource:
+                                properties:
+                                  container:
+                                    type: string
+                                  name:
+                                    type: string
+                                  target:
+                                    properties:
+                                      averageUtilization:
+                                        format: int32
+                                        type: integer
+                                      averageValue:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type:
+                                        type: string
+                                      value:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                    required:
+                                    - type
+                                    type: object
+                                required:
+                                - container
+                                - name
+                                - target
+                                type: object
+                              external:
+                                properties:
+                                  metric:
+                                    properties:
+                                      name:
+                                        type: string
+                                      selector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                    required:
+                                    - name
+                                    type: object
+                                  target:
+                                    properties:
+                                      averageUtilization:
+                                        format: int32
+                                        type: integer
+                                      averageValue:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type:
+                                        type: string
+                                      value:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                    required:
+                                    - type
+                                    type: object
+                                required:
+                                - metric
+                                - target
+                                type: object
+                              object:
+                                properties:
+                                  describedObject:
+                                    properties:
+                                      apiVersion:
+                                        type: string
+                                      kind:
+                                        type: string
+                                      name:
+                                        type: string
+                                    required:
+                                    - kind
+                                    - name
+                                    type: object
+                                  metric:
+                                    properties:
+                                      name:
+                                        type: string
+                                      selector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                    required:
+                                    - name
+                                    type: object
+                                  target:
+                                    properties:
+                                      averageUtilization:
+                                        format: int32
+                                        type: integer
+                                      averageValue:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type:
+                                        type: string
+                                      value:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                    required:
+                                    - type
+                                    type: object
+                                required:
+                                - describedObject
+                                - metric
+                                - target
+                                type: object
+                              pods:
+                                properties:
+                                  metric:
+                                    properties:
+                                      name:
+                                        type: string
+                                      selector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                    required:
+                                    - name
+                                    type: object
+                                  target:
+                                    properties:
+                                      averageUtilization:
+                                        format: int32
+                                        type: integer
+                                      averageValue:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type:
+                                        type: string
+                                      value:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                    required:
+                                    - type
+                                    type: object
+                                required:
+                                - metric
+                                - target
+                                type: object
+                              resource:
+                                properties:
+                                  name:
+                                    type: string
+                                  target:
+                                    properties:
+                                      averageUtilization:
+                                        format: int32
+                                        type: integer
+                                      averageValue:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type:
+                                        type: string
+                                      value:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                    required:
+                                    - type
+                                    type: object
+                                required:
+                                - name
+                                - target
+                                type: object
+                              type:
+                                type: string
+                            required:
+                            - type
+                            type: object
+                          type: array
+                        minReplicas:
+                          format: int32
+                          type: integer
+                        scaleTargetRef:
+                          properties:
+                            apiVersion:
+                              type: string
+                            kind:
+                              type: string
+                            name:
+                              type: string
+                          required:
+                          - kind
+                          - name
+                          type: object
+                      required:
+                      - maxReplicas
+                      - scaleTargetRef
+                      type: object
+                    image:
+                      type: string
+                    imagePullPolicy:
+                      type: string
+                    imagePullSecrets:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    ingress:
+                      properties:
+                        defaultBackend:
+                          properties:
+                            resource:
+                              properties:
+                                apiGroup:
+                                  type: string
+                                kind:
+                                  type: string
+                                name:
+                                  type: string
+                              required:
+                              - kind
+                              - name
+                              type: object
+                            service:
+                              properties:
+                                name:
+                                  type: string
+                                port:
+                                  properties:
+                                    name:
+                                      type: string
+                                    number:
+                                      format: int32
+                                      type: integer
+                                  type: object
+                              required:
+                              - name
+                              type: object
+                          type: object
+                        ingressClassName:
+                          type: string
+                        rules:
+                          items:
+                            properties:
+                              host:
+                                type: string
+                              http:
+                                properties:
+                                  paths:
+                                    items:
+                                      properties:
+                                        backend:
+                                          properties:
+                                            resource:
+                                              properties:
+                                                apiGroup:
+                                                  type: string
+                                                kind:
+                                                  type: string
+                                                name:
+                                                  type: string
+                                              required:
+                                              - kind
+                                              - name
+                                              type: object
+                                            service:
+                                              properties:
+                                                name:
+                                                  type: string
+                                                port:
+                                                  properties:
+                                                    name:
+                                                      type: string
+                                                    number:
+                                                      format: int32
+                                                      type: integer
+                                                  type: object
+                                              required:
+                                              - name
+                                              type: object
+                                          type: object
+                                        path:
+                                          type: string
+                                        pathType:
+                                          type: string
+                                      required:
+                                      - backend
+                                      type: object
+                                    type: array
+                                    x-kubernetes-list-type: atomic
+                                required:
+                                - paths
+                                type: object
+                            type: object
+                          type: array
+                          x-kubernetes-list-type: atomic
+                        tls:
+                          items:
+                            properties:
+                              hosts:
+                                items:
+                                  type: string
+                                type: array
+                                x-kubernetes-list-type: atomic
+                              secretName:
+                                type: string
+                            type: object
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      type: object
+                    ingressAnnotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    jvm.options:
+                      type: string
+                    kind:
+                      type: string
+                    lifecycle:
+                      properties:
+                        postStart:
+                          properties:
+                            exec:
+                              properties:
+                                command:
+                                  items:
+                                    type: string
+                                  type: array
+                              type: object
+                            httpGet:
+                              properties:
+                                host:
+                                  type: string
+                                httpHeaders:
+                                  items:
+                                    properties:
+                                      name:
+                                        type: string
+                                      value:
+                                        type: string
+                                    required:
+                                    - name
+                                    - value
+                                    type: object
+                                  type: array
+                                path:
+                                  type: string
+                                port:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  x-kubernetes-int-or-string: true
+                                scheme:
+                                  type: string
+                              required:
+                              - port
+                              type: object
+                            tcpSocket:
+                              properties:
+                                host:
+                                  type: string
+                                port:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  x-kubernetes-int-or-string: true
+                              required:
+                              - port
+                              type: object
+                          type: object
+                        preStop:
+                          properties:
+                            exec:
+                              properties:
+                                command:
+                                  items:
+                                    type: string
+                                  type: array
+                              type: object
+                            httpGet:
+                              properties:
+                                host:
+                                  type: string
+                                httpHeaders:
+                                  items:
+                                    properties:
+                                      name:
+                                        type: string
+                                      value:
+                                        type: string
+                                    required:
+                                    - name
+                                    - value
+                                    type: object
+                                  type: array
+                                path:
+                                  type: string
+                                port:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  x-kubernetes-int-or-string: true
+                                scheme:
+                                  type: string
+                              required:
+                              - port
+                              type: object
+                            tcpSocket:
+                              properties:
+                                host:
+                                  type: string
+                                port:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  x-kubernetes-int-or-string: true
+                              required:
+                              - port
+                              type: object
+                          type: object
+                      type: object
+                    livenessProbe:
+                      properties:
+                        exec:
+                          properties:
+                            command:
+                              items:
+                                type: string
+                              type: array
+                          type: object
+                        failureThreshold:
+                          format: int32
+                          type: integer
+                        httpGet:
+                          properties:
+                            host:
+                              type: string
+                            httpHeaders:
+                              items:
+                                properties:
+                                  name:
+                                    type: string
+                                  value:
+                                    type: string
+                                required:
+                                - name
+                                - value
+                                type: object
+                              type: array
+                            path:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                            scheme:
+                              type: string
+                          required:
+                          - port
+                          type: object
+                        initialDelaySeconds:
+                          format: int32
+                          type: integer
+                        periodSeconds:
+                          format: int32
+                          type: integer
+                        successThreshold:
+                          format: int32
+                          type: integer
+                        tcpSocket:
+                          properties:
+                            host:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          required:
+                          - port
+                          type: object
+                        timeoutSeconds:
+                          format: int32
+                          type: integer
+                      type: object
+                    log4j.config:
+                      type: string
+                    maxSurge:
+                      format: int32
+                      type: integer
+                    maxUnavailable:
+                      format: int32
+                      type: integer
+                    nodeConfigMountPath:
+                      type: string
+                    nodeType:
+                      type: string
+                    persistentVolumeClaim:
+                      items:
+                        properties:
+                          apiVersion:
+                            type: string
+                          kind:
+                            type: string
+                          metadata:
+                            properties:
+                              annotations:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              finalizers:
+                                items:
+                                  type: string
+                                type: array
+                              labels:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              name:
+                                type: string
+                              namespace:
+                                type: string
+                            type: object
+                          spec:
+                            properties:
+                              accessModes:
+                                items:
+                                  type: string
+                                type: array
+                              dataSource:
+                                properties:
+                                  apiGroup:
+                                    type: string
+                                  kind:
+                                    type: string
+                                  name:
+                                    type: string
+                                required:
+                                - kind
+                                - name
+                                type: object
+                              resources:
+                                properties:
+                                  limits:
+                                    additionalProperties:
+                                      anyOf:
+                                      - type: integer
+                                      - type: string
+                                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                      x-kubernetes-int-or-string: true
+                                    type: object
+                                  requests:
+                                    additionalProperties:
+                                      anyOf:
+                                      - type: integer
+                                      - type: string
+                                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                      x-kubernetes-int-or-string: true
+                                    type: object
+                                type: object
+                              selector:
+                                properties:
+                                  matchExpressions:
+                                    items:
+                                      properties:
+                                        key:
+                                          type: string
+                                        operator:
+                                          type: string
+                                        values:
+                                          items:
+                                            type: string
+                                          type: array
+                                      required:
+                                      - key
+                                      - operator
+                                      type: object
+                                    type: array
+                                  matchLabels:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
+                                type: object
+                              storageClassName:
+                                type: string
+                              volumeMode:
+                                type: string
+                              volumeName:
+                                type: string
+                            type: object
+                          status:
+                            properties:
+                              accessModes:
+                                items:
+                                  type: string
+                                type: array
+                              capacity:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                type: object
+                              conditions:
+                                items:
+                                  properties:
+                                    lastProbeTime:
+                                      format: date-time
+                                      type: string
+                                    lastTransitionTime:
+                                      format: date-time
+                                      type: string
+                                    message:
+                                      type: string
+                                    reason:
+                                      type: string
+                                    status:
+                                      type: string
+                                    type:
+                                      type: string
+                                  required:
+                                  - status
+                                  - type
+                                  type: object
+                                type: array
+                              phase:
+                                type: string
+                            type: object
+                        type: object
+                      type: array
+                    podAnnotations:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    podDisruptionBudgetSpec:
+                      properties:
+                        maxUnavailable:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          x-kubernetes-int-or-string: true
+                        minAvailable:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          x-kubernetes-int-or-string: true
+                        selector:
+                          properties:
+                            matchExpressions:
+                              items:
+                                properties:
+                                  key:
+                                    type: string
+                                  operator:
+                                    type: string
+                                  values:
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              type: object
+                          type: object
+                      type: object
+                    podLabels:
+                      additionalProperties:
+                        type: string
+                      type: object
+                    podManagementPolicy:
+                      type: string
+                    ports:
+                      items:
+                        properties:
+                          containerPort:
+                            format: int32
+                            type: integer
+                          hostIP:
+                            type: string
+                          hostPort:
+                            format: int32
+                            type: integer
+                          name:
+                            type: string
+                          protocol:
+                            default: TCP
+                            type: string
+                        required:
+                        - containerPort
+                        type: object
+                      type: array
+                    readinessProbe:
+                      properties:
+                        exec:
+                          properties:
+                            command:
+                              items:
+                                type: string
+                              type: array
+                          type: object
+                        failureThreshold:
+                          format: int32
+                          type: integer
+                        httpGet:
+                          properties:
+                            host:
+                              type: string
+                            httpHeaders:
+                              items:
+                                properties:
+                                  name:
+                                    type: string
+                                  value:
+                                    type: string
+                                required:
+                                - name
+                                - value
+                                type: object
+                              type: array
+                            path:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                            scheme:
+                              type: string
+                          required:
+                          - port
+                          type: object
+                        initialDelaySeconds:
+                          format: int32
+                          type: integer
+                        periodSeconds:
+                          format: int32
+                          type: integer
+                        successThreshold:
+                          format: int32
+                          type: integer
+                        tcpSocket:
+                          properties:
+                            host:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          required:
+                          - port
+                          type: object
+                        timeoutSeconds:
+                          format: int32
+                          type: integer
+                      type: object
+                    replicas:
+                      format: int32
+                      minimum: 0
+                      type: integer
+                    resources:
+                      properties:
+                        limits:
+                          additionalProperties:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          type: object
+                        requests:
+                          additionalProperties:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          type: object
+                      type: object
+                    runtime.properties:
+                      type: string
+                    securityContext:
+                      properties:
+                        fsGroup:
+                          format: int64
+                          type: integer
+                        fsGroupChangePolicy:
+                          type: string
+                        runAsGroup:
+                          format: int64
+                          type: integer
+                        runAsNonRoot:
+                          type: boolean
+                        runAsUser:
+                          format: int64
+                          type: integer
+                        seLinuxOptions:
+                          properties:
+                            level:
+                              type: string
+                            role:
+                              type: string
+                            type:
+                              type: string
+                            user:
+                              type: string
+                          type: object
+                        seccompProfile:
+                          properties:
+                            localhostProfile:
+                              type: string
+                            type:
+                              type: string
+                          required:
+                          - type
+                          type: object
+                        supplementalGroups:
+                          items:
+                            format: int64
+                            type: integer
+                          type: array
+                        sysctls:
+                          items:
+                            properties:
+                              name:
+                                type: string
+                              value:
+                                type: string
+                            required:
+                            - name
+                            - value
+                            type: object
+                          type: array
+                        windowsOptions:
+                          properties:
+                            gmsaCredentialSpec:
+                              type: string
+                            gmsaCredentialSpecName:
+                              type: string
+                            runAsUserName:
+                              type: string
+                          type: object
+                      type: object
+                    services:
+                      items:
+                        properties:
+                          apiVersion:
+                            type: string
+                          kind:
+                            type: string
+                          metadata:
+                            properties:
+                              annotations:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              finalizers:
+                                items:
+                                  type: string
+                                type: array
+                              labels:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              name:
+                                type: string
+                              namespace:
+                                type: string
+                            type: object
+                          spec:
+                            properties:
+                              allocateLoadBalancerNodePorts:
+                                type: boolean
+                              clusterIP:
+                                type: string
+                              clusterIPs:
+                                items:
+                                  type: string
+                                type: array
+                                x-kubernetes-list-type: atomic
+                              externalIPs:
+                                items:
+                                  type: string
+                                type: array
+                              externalName:
+                                type: string
+                              externalTrafficPolicy:
+                                type: string
+                              healthCheckNodePort:
+                                format: int32
+                                type: integer
+                              ipFamilies:
+                                items:
+                                  type: string
+                                type: array
+                                x-kubernetes-list-type: atomic
+                              ipFamilyPolicy:
+                                type: string
+                              loadBalancerIP:
+                                type: string
+                              loadBalancerSourceRanges:
+                                items:
+                                  type: string
+                                type: array
+                              ports:
+                                items:
+                                  properties:
+                                    appProtocol:
+                                      type: string
+                                    name:
+                                      type: string
+                                    nodePort:
+                                      format: int32
+                                      type: integer
+                                    port:
+                                      format: int32
+                                      type: integer
+                                    protocol:
+                                      default: TCP
+                                      type: string
+                                    targetPort:
+                                      anyOf:
+                                      - type: integer
+                                      - type: string
+                                      x-kubernetes-int-or-string: true
+                                  required:
+                                  - port
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - port
+                                - protocol
+                                x-kubernetes-list-type: map
+                              publishNotReadyAddresses:
+                                type: boolean
+                              selector:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              sessionAffinity:
+                                type: string
+                              sessionAffinityConfig:
+                                properties:
+                                  clientIP:
+                                    properties:
+                                      timeoutSeconds:
+                                        format: int32
+                                        type: integer
+                                    type: object
+                                type: object
+                              topologyKeys:
+                                items:
+                                  type: string
+                                type: array
+                              type:
+                                type: string
+                            type: object
+                          status:
+                            properties:
+                              conditions:
+                                items:
+                                  properties:
+                                    lastTransitionTime:
+                                      format: date-time
+                                      type: string
+                                    message:
+                                      maxLength: 32768
+                                      type: string
+                                    observedGeneration:
+                                      format: int64
+                                      minimum: 0
+                                      type: integer
+                                    reason:
+                                      maxLength: 1024
+                                      minLength: 1
+                                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                      type: string
+                                    status:
+                                      enum:
+                                      - "True"
+                                      - "False"
+                                      - Unknown
+                                      type: string
+                                    type:
+                                      maxLength: 316
+                                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                      type: string
+                                  required:
+                                  - lastTransitionTime
+                                  - message
+                                  - reason
+                                  - status
+                                  - type
+                                  type: object
+                                type: array
+                                x-kubernetes-list-map-keys:
+                                - type
+                                x-kubernetes-list-type: map
+                              loadBalancer:
+                                properties:
+                                  ingress:
+                                    items:
+                                      properties:
+                                        hostname:
+                                          type: string
+                                        ip:
+                                          type: string
+                                        ports:
+                                          items:
+                                            properties:
+                                              error:
+                                                maxLength: 316
+                                                pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                                type: string
+                                              port:
+                                                format: int32
+                                                type: integer
+                                              protocol:
+                                                default: TCP
+                                                type: string
+                                            required:
+                                            - port
+                                            - protocol
+                                            type: object
+                                          type: array
+                                          x-kubernetes-list-type: atomic
+                                      type: object
+                                    type: array
+                                type: object
+                            type: object
+                        type: object
+                      type: array
+                    startUpProbes:
+                      properties:
+                        exec:
+                          properties:
+                            command:
+                              items:
+                                type: string
+                              type: array
+                          type: object
+                        failureThreshold:
+                          format: int32
+                          type: integer
+                        httpGet:
+                          properties:
+                            host:
+                              type: string
+                            httpHeaders:
+                              items:
+                                properties:
+                                  name:
+                                    type: string
+                                  value:
+                                    type: string
+                                required:
+                                - name
+                                - value
+                                type: object
+                              type: array
+                            path:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                            scheme:
+                              type: string
+                          required:
+                          - port
+                          type: object
+                        initialDelaySeconds:
+                          format: int32
+                          type: integer
+                        periodSeconds:
+                          format: int32
+                          type: integer
+                        successThreshold:
+                          format: int32
+                          type: integer
+                        tcpSocket:
+                          properties:
+                            host:
+                              type: string
+                            port:
+                              anyOf:
+                              - type: integer
+                              - type: string
+                              x-kubernetes-int-or-string: true
+                          required:
+                          - port
+                          type: object
+                        timeoutSeconds:
+                          format: int32
+                          type: integer
+                      type: object
+                    terminationGracePeriodSeconds:
+                      format: int64
+                      type: integer
+                    tolerations:
+                      items:
+                        properties:
+                          effect:
+                            type: string
+                          key:
+                            type: string
+                          operator:
+                            type: string
+                          tolerationSeconds:
+                            format: int64
+                            type: integer
+                          value:
+                            type: string
+                        type: object
+                      type: array
+                    topologySpreadConstraints:
+                      items:
+                        properties:
+                          labelSelector:
+                            properties:
+                              matchExpressions:
+                                items:
+                                  properties:
+                                    key:
+                                      type: string
+                                    operator:
+                                      type: string
+                                    values:
+                                      items:
+                                        type: string
+                                      type: array
+                                  required:
+                                  - key
+                                  - operator
+                                  type: object
+                                type: array
+                              matchLabels:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                            type: object
+                          maxSkew:
+                            format: int32
+                            type: integer
+                          topologyKey:
+                            type: string
+                          whenUnsatisfiable:
+                            type: string
+                        required:
+                        - maxSkew
+                        - topologyKey
+                        - whenUnsatisfiable
+                        type: object
+                      type: array
+                    updateStrategy:
+                      properties:
+                        rollingUpdate:
+                          properties:
+                            partition:
+                              format: int32
+                              type: integer
+                          type: object
+                        type:
+                          type: string
+                      type: object
+                    volumeClaimTemplates:
+                      items:
+                        properties:
+                          apiVersion:
+                            type: string
+                          kind:
+                            type: string
+                          metadata:
+                            properties:
+                              annotations:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              finalizers:
+                                items:
+                                  type: string
+                                type: array
+                              labels:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              name:
+                                type: string
+                              namespace:
+                                type: string
+                            type: object
+                          spec:
+                            properties:
+                              accessModes:
+                                items:
+                                  type: string
+                                type: array
+                              dataSource:
+                                properties:
+                                  apiGroup:
+                                    type: string
+                                  kind:
+                                    type: string
+                                  name:
+                                    type: string
+                                required:
+                                - kind
+                                - name
+                                type: object
+                              resources:
+                                properties:
+                                  limits:
+                                    additionalProperties:
+                                      anyOf:
+                                      - type: integer
+                                      - type: string
+                                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                      x-kubernetes-int-or-string: true
+                                    type: object
+                                  requests:
+                                    additionalProperties:
+                                      anyOf:
+                                      - type: integer
+                                      - type: string
+                                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                      x-kubernetes-int-or-string: true
+                                    type: object
+                                type: object
+                              selector:
+                                properties:
+                                  matchExpressions:
+                                    items:
+                                      properties:
+                                        key:
+                                          type: string
+                                        operator:
+                                          type: string
+                                        values:
+                                          items:
+                                            type: string
+                                          type: array
+                                      required:
+                                      - key
+                                      - operator
+                                      type: object
+                                    type: array
+                                  matchLabels:
+                                    additionalProperties:
+                                      type: string
+                                    type: object
+                                type: object
+                              storageClassName:
+                                type: string
+                              volumeMode:
+                                type: string
+                              volumeName:
+                                type: string
+                            type: object
+                          status:
+                            properties:
+                              accessModes:
+                                items:
+                                  type: string
+                                type: array
+                              capacity:
+                                additionalProperties:
+                                  anyOf:
+                                  - type: integer
+                                  - type: string
+                                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                  x-kubernetes-int-or-string: true
+                                type: object
+                              conditions:
+                                items:
+                                  properties:
+                                    lastProbeTime:
+                                      format: date-time
+                                      type: string
+                                    lastTransitionTime:
+                                      format: date-time
+                                      type: string
+                                    message:
+                                      type: string
+                                    reason:
+                                      type: string
+                                    status:
+                                      type: string
+                                    type:
+                                      type: string
+                                  required:
+                                  - status
+                                  - type
+                                  type: object
+                                type: array
+                              phase:
+                                type: string
+                            type: object
+                        type: object
+                      type: array
+                    volumeMounts:
+                      items:
+                        properties:
+                          mountPath:
+                            type: string
+                          mountPropagation:
+                            type: string
+                          name:
+                            type: string
+                          readOnly:
+                            type: boolean
+                          subPath:
+                            type: string
+                          subPathExpr:
+                            type: string
+                        required:
+                        - mountPath
+                        - name
+                        type: object
+                      type: array
+                    volumes:
+                      items:
+                        properties:
+                          awsElasticBlockStore:
+                            properties:
+                              fsType:
+                                type: string
+                              partition:
+                                format: int32
+                                type: integer
+                              readOnly:
+                                type: boolean
+                              volumeID:
+                                type: string
+                            required:
+                            - volumeID
+                            type: object
+                          azureDisk:
+                            properties:
+                              cachingMode:
+                                type: string
+                              diskName:
+                                type: string
+                              diskURI:
+                                type: string
+                              fsType:
+                                type: string
+                              kind:
+                                type: string
+                              readOnly:
+                                type: boolean
+                            required:
+                            - diskName
+                            - diskURI
+                            type: object
+                          azureFile:
+                            properties:
+                              readOnly:
+                                type: boolean
+                              secretName:
+                                type: string
+                              shareName:
+                                type: string
+                            required:
+                            - secretName
+                            - shareName
+                            type: object
+                          cephfs:
+                            properties:
+                              monitors:
+                                items:
+                                  type: string
+                                type: array
+                              path:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              secretFile:
+                                type: string
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              user:
+                                type: string
+                            required:
+                            - monitors
+                            type: object
+                          cinder:
+                            properties:
+                              fsType:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              volumeID:
+                                type: string
+                            required:
+                            - volumeID
+                            type: object
+                          configMap:
+                            properties:
+                              defaultMode:
+                                format: int32
+                                type: integer
+                              items:
+                                items:
+                                  properties:
+                                    key:
+                                      type: string
+                                    mode:
+                                      format: int32
+                                      type: integer
+                                    path:
+                                      type: string
+                                  required:
+                                  - key
+                                  - path
+                                  type: object
+                                type: array
+                              name:
+                                type: string
+                              optional:
+                                type: boolean
+                            type: object
+                          csi:
+                            properties:
+                              driver:
+                                type: string
+                              fsType:
+                                type: string
+                              nodePublishSecretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              readOnly:
+                                type: boolean
+                              volumeAttributes:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                            required:
+                            - driver
+                            type: object
+                          downwardAPI:
+                            properties:
+                              defaultMode:
+                                format: int32
+                                type: integer
+                              items:
+                                items:
+                                  properties:
+                                    fieldRef:
+                                      properties:
+                                        apiVersion:
+                                          type: string
+                                        fieldPath:
+                                          type: string
+                                      required:
+                                      - fieldPath
+                                      type: object
+                                    mode:
+                                      format: int32
+                                      type: integer
+                                    path:
+                                      type: string
+                                    resourceFieldRef:
+                                      properties:
+                                        containerName:
+                                          type: string
+                                        divisor:
+                                          anyOf:
+                                          - type: integer
+                                          - type: string
+                                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                          x-kubernetes-int-or-string: true
+                                        resource:
+                                          type: string
+                                      required:
+                                      - resource
+                                      type: object
+                                  required:
+                                  - path
+                                  type: object
+                                type: array
+                            type: object
+                          emptyDir:
+                            properties:
+                              medium:
+                                type: string
+                              sizeLimit:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
+                            type: object
+                          ephemeral:
+                            properties:
+                              readOnly:
+                                type: boolean
+                              volumeClaimTemplate:
+                                properties:
+                                  metadata:
+                                    properties:
+                                      annotations:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                      finalizers:
+                                        items:
+                                          type: string
+                                        type: array
+                                      labels:
+                                        additionalProperties:
+                                          type: string
+                                        type: object
+                                      name:
+                                        type: string
+                                      namespace:
+                                        type: string
+                                    type: object
+                                  spec:
+                                    properties:
+                                      accessModes:
+                                        items:
+                                          type: string
+                                        type: array
+                                      dataSource:
+                                        properties:
+                                          apiGroup:
+                                            type: string
+                                          kind:
+                                            type: string
+                                          name:
+                                            type: string
+                                        required:
+                                        - kind
+                                        - name
+                                        type: object
+                                      resources:
+                                        properties:
+                                          limits:
+                                            additionalProperties:
+                                              anyOf:
+                                              - type: integer
+                                              - type: string
+                                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                              x-kubernetes-int-or-string: true
+                                            type: object
+                                          requests:
+                                            additionalProperties:
+                                              anyOf:
+                                              - type: integer
+                                              - type: string
+                                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                              x-kubernetes-int-or-string: true
+                                            type: object
+                                        type: object
+                                      selector:
+                                        properties:
+                                          matchExpressions:
+                                            items:
+                                              properties:
+                                                key:
+                                                  type: string
+                                                operator:
+                                                  type: string
+                                                values:
+                                                  items:
+                                                    type: string
+                                                  type: array
+                                              required:
+                                              - key
+                                              - operator
+                                              type: object
+                                            type: array
+                                          matchLabels:
+                                            additionalProperties:
+                                              type: string
+                                            type: object
+                                        type: object
+                                      storageClassName:
+                                        type: string
+                                      volumeMode:
+                                        type: string
+                                      volumeName:
+                                        type: string
+                                    type: object
+                                required:
+                                - spec
+                                type: object
+                            type: object
+                          fc:
+                            properties:
+                              fsType:
+                                type: string
+                              lun:
+                                format: int32
+                                type: integer
+                              readOnly:
+                                type: boolean
+                              targetWWNs:
+                                items:
+                                  type: string
+                                type: array
+                              wwids:
+                                items:
+                                  type: string
+                                type: array
+                            type: object
+                          flexVolume:
+                            properties:
+                              driver:
+                                type: string
+                              fsType:
+                                type: string
+                              options:
+                                additionalProperties:
+                                  type: string
+                                type: object
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                            required:
+                            - driver
+                            type: object
+                          flocker:
+                            properties:
+                              datasetName:
+                                type: string
+                              datasetUUID:
+                                type: string
+                            type: object
+                          gcePersistentDisk:
+                            properties:
+                              fsType:
+                                type: string
+                              partition:
+                                format: int32
+                                type: integer
+                              pdName:
+                                type: string
+                              readOnly:
+                                type: boolean
+                            required:
+                            - pdName
+                            type: object
+                          gitRepo:
+                            properties:
+                              directory:
+                                type: string
+                              repository:
+                                type: string
+                              revision:
+                                type: string
+                            required:
+                            - repository
+                            type: object
+                          glusterfs:
+                            properties:
+                              endpoints:
+                                type: string
+                              path:
+                                type: string
+                              readOnly:
+                                type: boolean
+                            required:
+                            - endpoints
+                            - path
+                            type: object
+                          hostPath:
+                            properties:
+                              path:
+                                type: string
+                              type:
+                                type: string
+                            required:
+                            - path
+                            type: object
+                          iscsi:
+                            properties:
+                              chapAuthDiscovery:
+                                type: boolean
+                              chapAuthSession:
+                                type: boolean
+                              fsType:
+                                type: string
+                              initiatorName:
+                                type: string
+                              iqn:
+                                type: string
+                              iscsiInterface:
+                                type: string
+                              lun:
+                                format: int32
+                                type: integer
+                              portals:
+                                items:
+                                  type: string
+                                type: array
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              targetPortal:
+                                type: string
+                            required:
+                            - iqn
+                            - lun
+                            - targetPortal
+                            type: object
+                          name:
+                            type: string
+                          nfs:
+                            properties:
+                              path:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              server:
+                                type: string
+                            required:
+                            - path
+                            - server
+                            type: object
+                          persistentVolumeClaim:
+                            properties:
+                              claimName:
+                                type: string
+                              readOnly:
+                                type: boolean
+                            required:
+                            - claimName
+                            type: object
+                          photonPersistentDisk:
+                            properties:
+                              fsType:
+                                type: string
+                              pdID:
+                                type: string
+                            required:
+                            - pdID
+                            type: object
+                          portworxVolume:
+                            properties:
+                              fsType:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              volumeID:
+                                type: string
+                            required:
+                            - volumeID
+                            type: object
+                          projected:
+                            properties:
+                              defaultMode:
+                                format: int32
+                                type: integer
+                              sources:
+                                items:
+                                  properties:
+                                    configMap:
+                                      properties:
+                                        items:
+                                          items:
+                                            properties:
+                                              key:
+                                                type: string
+                                              mode:
+                                                format: int32
+                                                type: integer
+                                              path:
+                                                type: string
+                                            required:
+                                            - key
+                                            - path
+                                            type: object
+                                          type: array
+                                        name:
+                                          type: string
+                                        optional:
+                                          type: boolean
+                                      type: object
+                                    downwardAPI:
+                                      properties:
+                                        items:
+                                          items:
+                                            properties:
+                                              fieldRef:
+                                                properties:
+                                                  apiVersion:
+                                                    type: string
+                                                  fieldPath:
+                                                    type: string
+                                                required:
+                                                - fieldPath
+                                                type: object
+                                              mode:
+                                                format: int32
+                                                type: integer
+                                              path:
+                                                type: string
+                                              resourceFieldRef:
+                                                properties:
+                                                  containerName:
+                                                    type: string
+                                                  divisor:
+                                                    anyOf:
+                                                    - type: integer
+                                                    - type: string
+                                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                                    x-kubernetes-int-or-string: true
+                                                  resource:
+                                                    type: string
+                                                required:
+                                                - resource
+                                                type: object
+                                            required:
+                                            - path
+                                            type: object
+                                          type: array
+                                      type: object
+                                    secret:
+                                      properties:
+                                        items:
+                                          items:
+                                            properties:
+                                              key:
+                                                type: string
+                                              mode:
+                                                format: int32
+                                                type: integer
+                                              path:
+                                                type: string
+                                            required:
+                                            - key
+                                            - path
+                                            type: object
+                                          type: array
+                                        name:
+                                          type: string
+                                        optional:
+                                          type: boolean
+                                      type: object
+                                    serviceAccountToken:
+                                      properties:
+                                        audience:
+                                          type: string
+                                        expirationSeconds:
+                                          format: int64
+                                          type: integer
+                                        path:
+                                          type: string
+                                      required:
+                                      - path
+                                      type: object
+                                  type: object
+                                type: array
+                            type: object
+                          quobyte:
+                            properties:
+                              group:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              registry:
+                                type: string
+                              tenant:
+                                type: string
+                              user:
+                                type: string
+                              volume:
+                                type: string
+                            required:
+                            - registry
+                            - volume
+                            type: object
+                          rbd:
+                            properties:
+                              fsType:
+                                type: string
+                              image:
+                                type: string
+                              keyring:
+                                type: string
+                              monitors:
+                                items:
+                                  type: string
+                                type: array
+                              pool:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              user:
+                                type: string
+                            required:
+                            - image
+                            - monitors
+                            type: object
+                          scaleIO:
+                            properties:
+                              fsType:
+                                type: string
+                              gateway:
+                                type: string
+                              protectionDomain:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              sslEnabled:
+                                type: boolean
+                              storageMode:
+                                type: string
+                              storagePool:
+                                type: string
+                              system:
+                                type: string
+                              volumeName:
+                                type: string
+                            required:
+                            - gateway
+                            - secretRef
+                            - system
+                            type: object
+                          secret:
+                            properties:
+                              defaultMode:
+                                format: int32
+                                type: integer
+                              items:
+                                items:
+                                  properties:
+                                    key:
+                                      type: string
+                                    mode:
+                                      format: int32
+                                      type: integer
+                                    path:
+                                      type: string
+                                  required:
+                                  - key
+                                  - path
+                                  type: object
+                                type: array
+                              optional:
+                                type: boolean
+                              secretName:
+                                type: string
+                            type: object
+                          storageos:
+                            properties:
+                              fsType:
+                                type: string
+                              readOnly:
+                                type: boolean
+                              secretRef:
+                                properties:
+                                  name:
+                                    type: string
+                                type: object
+                              volumeName:
+                                type: string
+                              volumeNamespace:
+                                type: string
+                            type: object
+                          vsphereVolume:
+                            properties:
+                              fsType:
+                                type: string
+                              storagePolicyID:
+                                type: string
+                              storagePolicyName:
+                                type: string
+                              volumePath:
+                                type: string
+                            required:
+                            - volumePath
+                            type: object
+                        required:
+                        - name
+                        type: object
+                      type: array
+                  required:
+                  - druid.port
+                  - nodeConfigMountPath
+                  - nodeType
+                  - replicas
+                  - runtime.properties
+                  type: object
+                type: object
+              podAnnotations:
+                additionalProperties:
+                  type: string
+                type: object
+              podLabels:
+                additionalProperties:
+                  type: string
+                type: object
+              podManagementPolicy:
+                type: string
+              readinessProbe:
+                properties:
+                  exec:
+                    properties:
+                      command:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  failureThreshold:
+                    format: int32
+                    type: integer
+                  httpGet:
+                    properties:
+                      host:
+                        type: string
+                      httpHeaders:
+                        items:
+                          properties:
+                            name:
+                              type: string
+                            value:
+                              type: string
+                          required:
+                          - name
+                          - value
+                          type: object
+                        type: array
+                      path:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                      scheme:
+                        type: string
+                    required:
+                    - port
+                    type: object
+                  initialDelaySeconds:
+                    format: int32
+                    type: integer
+                  periodSeconds:
+                    format: int32
+                    type: integer
+                  successThreshold:
+                    format: int32
+                    type: integer
+                  tcpSocket:
+                    properties:
+                      host:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    required:
+                    - port
+                    type: object
+                  timeoutSeconds:
+                    format: int32
+                    type: integer
+                type: object
+              rollingDeploy:
+                type: boolean
+              securityContext:
+                properties:
+                  fsGroup:
+                    format: int64
+                    type: integer
+                  fsGroupChangePolicy:
+                    type: string
+                  runAsGroup:
+                    format: int64
+                    type: integer
+                  runAsNonRoot:
+                    type: boolean
+                  runAsUser:
+                    format: int64
+                    type: integer
+                  seLinuxOptions:
+                    properties:
+                      level:
+                        type: string
+                      role:
+                        type: string
+                      type:
+                        type: string
+                      user:
+                        type: string
+                    type: object
+                  seccompProfile:
+                    properties:
+                      localhostProfile:
+                        type: string
+                      type:
+                        type: string
+                    required:
+                    - type
+                    type: object
+                  supplementalGroups:
+                    items:
+                      format: int64
+                      type: integer
+                    type: array
+                  sysctls:
+                    items:
+                      properties:
+                        name:
+                          type: string
+                        value:
+                          type: string
+                      required:
+                      - name
+                      - value
+                      type: object
+                    type: array
+                  windowsOptions:
+                    properties:
+                      gmsaCredentialSpec:
+                        type: string
+                      gmsaCredentialSpecName:
+                        type: string
+                      runAsUserName:
+                        type: string
+                    type: object
+                type: object
+              serviceAccount:
+                type: string
+              services:
+                items:
+                  properties:
+                    apiVersion:
+                      type: string
+                    kind:
+                      type: string
+                    metadata:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        finalizers:
+                          items:
+                            type: string
+                          type: array
+                        labels:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      type: object
+                    spec:
+                      properties:
+                        allocateLoadBalancerNodePorts:
+                          type: boolean
+                        clusterIP:
+                          type: string
+                        clusterIPs:
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                        externalIPs:
+                          items:
+                            type: string
+                          type: array
+                        externalName:
+                          type: string
+                        externalTrafficPolicy:
+                          type: string
+                        healthCheckNodePort:
+                          format: int32
+                          type: integer
+                        ipFamilies:
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                        ipFamilyPolicy:
+                          type: string
+                        loadBalancerIP:
+                          type: string
+                        loadBalancerSourceRanges:
+                          items:
+                            type: string
+                          type: array
+                        ports:
+                          items:
+                            properties:
+                              appProtocol:
+                                type: string
+                              name:
+                                type: string
+                              nodePort:
+                                format: int32
+                                type: integer
+                              port:
+                                format: int32
+                                type: integer
+                              protocol:
+                                default: TCP
+                                type: string
+                              targetPort:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                x-kubernetes-int-or-string: true
+                            required:
+                            - port
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - port
+                          - protocol
+                          x-kubernetes-list-type: map
+                        publishNotReadyAddresses:
+                          type: boolean
+                        selector:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        sessionAffinity:
+                          type: string
+                        sessionAffinityConfig:
+                          properties:
+                            clientIP:
+                              properties:
+                                timeoutSeconds:
+                                  format: int32
+                                  type: integer
+                              type: object
+                          type: object
+                        topologyKeys:
+                          items:
+                            type: string
+                          type: array
+                        type:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        conditions:
+                          items:
+                            properties:
+                              lastTransitionTime:
+                                format: date-time
+                                type: string
+                              message:
+                                maxLength: 32768
+                                type: string
+                              observedGeneration:
+                                format: int64
+                                minimum: 0
+                                type: integer
+                              reason:
+                                maxLength: 1024
+                                minLength: 1
+                                pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
+                                type: string
+                              status:
+                                enum:
+                                - "True"
+                                - "False"
+                                - Unknown
+                                type: string
+                              type:
+                                maxLength: 316
+                                pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                type: string
+                            required:
+                            - lastTransitionTime
+                            - message
+                            - reason
+                            - status
+                            - type
+                            type: object
+                          type: array
+                          x-kubernetes-list-map-keys:
+                          - type
+                          x-kubernetes-list-type: map
+                        loadBalancer:
+                          properties:
+                            ingress:
+                              items:
+                                properties:
+                                  hostname:
+                                    type: string
+                                  ip:
+                                    type: string
+                                  ports:
+                                    items:
+                                      properties:
+                                        error:
+                                          maxLength: 316
+                                          pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
+                                          type: string
+                                        port:
+                                          format: int32
+                                          type: integer
+                                        protocol:
+                                          default: TCP
+                                          type: string
+                                      required:
+                                      - port
+                                      - protocol
+                                      type: object
+                                    type: array
+                                    x-kubernetes-list-type: atomic
+                                type: object
+                              type: array
+                          type: object
+                      type: object
+                  type: object
+                type: array
+              startScript:
+                type: string
+              startUpProbes:
+                properties:
+                  exec:
+                    properties:
+                      command:
+                        items:
+                          type: string
+                        type: array
+                    type: object
+                  failureThreshold:
+                    format: int32
+                    type: integer
+                  httpGet:
+                    properties:
+                      host:
+                        type: string
+                      httpHeaders:
+                        items:
+                          properties:
+                            name:
+                              type: string
+                            value:
+                              type: string
+                          required:
+                          - name
+                          - value
+                          type: object
+                        type: array
+                      path:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                      scheme:
+                        type: string
+                    required:
+                    - port
+                    type: object
+                  initialDelaySeconds:
+                    format: int32
+                    type: integer
+                  periodSeconds:
+                    format: int32
+                    type: integer
+                  successThreshold:
+                    format: int32
+                    type: integer
+                  tcpSocket:
+                    properties:
+                      host:
+                        type: string
+                      port:
+                        anyOf:
+                        - type: integer
+                        - type: string
+                        x-kubernetes-int-or-string: true
+                    required:
+                    - port
+                    type: object
+                  timeoutSeconds:
+                    format: int32
+                    type: integer
+                type: object
+              tolerations:
+                items:
+                  properties:
+                    effect:
+                      type: string
+                    key:
+                      type: string
+                    operator:
+                      type: string
+                    tolerationSeconds:
+                      format: int64
+                      type: integer
+                    value:
+                      type: string
+                  type: object
+                type: array
+              updateStrategy:
+                properties:
+                  rollingUpdate:
+                    properties:
+                      partition:
+                        format: int32
+                        type: integer
+                    type: object
+                  type:
+                    type: string
+                type: object
+              volumeClaimTemplates:
+                items:
+                  properties:
+                    apiVersion:
+                      type: string
+                    kind:
+                      type: string
+                    metadata:
+                      properties:
+                        annotations:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        finalizers:
+                          items:
+                            type: string
+                          type: array
+                        labels:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        name:
+                          type: string
+                        namespace:
+                          type: string
+                      type: object
+                    spec:
+                      properties:
+                        accessModes:
+                          items:
+                            type: string
+                          type: array
+                        dataSource:
+                          properties:
+                            apiGroup:
+                              type: string
+                            kind:
+                              type: string
+                            name:
+                              type: string
+                          required:
+                          - kind
+                          - name
+                          type: object
+                        resources:
+                          properties:
+                            limits:
+                              additionalProperties:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
+                              type: object
+                            requests:
+                              additionalProperties:
+                                anyOf:
+                                - type: integer
+                                - type: string
+                                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                x-kubernetes-int-or-string: true
+                              type: object
+                          type: object
+                        selector:
+                          properties:
+                            matchExpressions:
+                              items:
+                                properties:
+                                  key:
+                                    type: string
+                                  operator:
+                                    type: string
+                                  values:
+                                    items:
+                                      type: string
+                                    type: array
+                                required:
+                                - key
+                                - operator
+                                type: object
+                              type: array
+                            matchLabels:
+                              additionalProperties:
+                                type: string
+                              type: object
+                          type: object
+                        storageClassName:
+                          type: string
+                        volumeMode:
+                          type: string
+                        volumeName:
+                          type: string
+                      type: object
+                    status:
+                      properties:
+                        accessModes:
+                          items:
+                            type: string
+                          type: array
+                        capacity:
+                          additionalProperties:
+                            anyOf:
+                            - type: integer
+                            - type: string
+                            pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                            x-kubernetes-int-or-string: true
+                          type: object
+                        conditions:
+                          items:
+                            properties:
+                              lastProbeTime:
+                                format: date-time
+                                type: string
+                              lastTransitionTime:
+                                format: date-time
+                                type: string
+                              message:
+                                type: string
+                              reason:
+                                type: string
+                              status:
+                                type: string
+                              type:
+                                type: string
+                            required:
+                            - status
+                            - type
+                            type: object
+                          type: array
+                        phase:
+                          type: string
+                      type: object
+                  type: object
+                type: array
+              volumeMounts:
+                items:
+                  properties:
+                    mountPath:
+                      type: string
+                    mountPropagation:
+                      type: string
+                    name:
+                      type: string
+                    readOnly:
+                      type: boolean
+                    subPath:
+                      type: string
+                    subPathExpr:
+                      type: string
+                  required:
+                  - mountPath
+                  - name
+                  type: object
+                type: array
+              volumes:
+                items:
+                  properties:
+                    awsElasticBlockStore:
+                      properties:
+                        fsType:
+                          type: string
+                        partition:
+                          format: int32
+                          type: integer
+                        readOnly:
+                          type: boolean
+                        volumeID:
+                          type: string
+                      required:
+                      - volumeID
+                      type: object
+                    azureDisk:
+                      properties:
+                        cachingMode:
+                          type: string
+                        diskName:
+                          type: string
+                        diskURI:
+                          type: string
+                        fsType:
+                          type: string
+                        kind:
+                          type: string
+                        readOnly:
+                          type: boolean
+                      required:
+                      - diskName
+                      - diskURI
+                      type: object
+                    azureFile:
+                      properties:
+                        readOnly:
+                          type: boolean
+                        secretName:
+                          type: string
+                        shareName:
+                          type: string
+                      required:
+                      - secretName
+                      - shareName
+                      type: object
+                    cephfs:
+                      properties:
+                        monitors:
+                          items:
+                            type: string
+                          type: array
+                        path:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        secretFile:
+                          type: string
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        user:
+                          type: string
+                      required:
+                      - monitors
+                      type: object
+                    cinder:
+                      properties:
+                        fsType:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        volumeID:
+                          type: string
+                      required:
+                      - volumeID
+                      type: object
+                    configMap:
+                      properties:
+                        defaultMode:
+                          format: int32
+                          type: integer
+                        items:
+                          items:
+                            properties:
+                              key:
+                                type: string
+                              mode:
+                                format: int32
+                                type: integer
+                              path:
+                                type: string
+                            required:
+                            - key
+                            - path
+                            type: object
+                          type: array
+                        name:
+                          type: string
+                        optional:
+                          type: boolean
+                      type: object
+                    csi:
+                      properties:
+                        driver:
+                          type: string
+                        fsType:
+                          type: string
+                        nodePublishSecretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        readOnly:
+                          type: boolean
+                        volumeAttributes:
+                          additionalProperties:
+                            type: string
+                          type: object
+                      required:
+                      - driver
+                      type: object
+                    downwardAPI:
+                      properties:
+                        defaultMode:
+                          format: int32
+                          type: integer
+                        items:
+                          items:
+                            properties:
+                              fieldRef:
+                                properties:
+                                  apiVersion:
+                                    type: string
+                                  fieldPath:
+                                    type: string
+                                required:
+                                - fieldPath
+                                type: object
+                              mode:
+                                format: int32
+                                type: integer
+                              path:
+                                type: string
+                              resourceFieldRef:
+                                properties:
+                                  containerName:
+                                    type: string
+                                  divisor:
+                                    anyOf:
+                                    - type: integer
+                                    - type: string
+                                    pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                    x-kubernetes-int-or-string: true
+                                  resource:
+                                    type: string
+                                required:
+                                - resource
+                                type: object
+                            required:
+                            - path
+                            type: object
+                          type: array
+                      type: object
+                    emptyDir:
+                      properties:
+                        medium:
+                          type: string
+                        sizeLimit:
+                          anyOf:
+                          - type: integer
+                          - type: string
+                          pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                          x-kubernetes-int-or-string: true
+                      type: object
+                    ephemeral:
+                      properties:
+                        readOnly:
+                          type: boolean
+                        volumeClaimTemplate:
+                          properties:
+                            metadata:
+                              properties:
+                                annotations:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                                finalizers:
+                                  items:
+                                    type: string
+                                  type: array
+                                labels:
+                                  additionalProperties:
+                                    type: string
+                                  type: object
+                                name:
+                                  type: string
+                                namespace:
+                                  type: string
+                              type: object
+                            spec:
+                              properties:
+                                accessModes:
+                                  items:
+                                    type: string
+                                  type: array
+                                dataSource:
+                                  properties:
+                                    apiGroup:
+                                      type: string
+                                    kind:
+                                      type: string
+                                    name:
+                                      type: string
+                                  required:
+                                  - kind
+                                  - name
+                                  type: object
+                                resources:
+                                  properties:
+                                    limits:
+                                      additionalProperties:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type: object
+                                    requests:
+                                      additionalProperties:
+                                        anyOf:
+                                        - type: integer
+                                        - type: string
+                                        pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                        x-kubernetes-int-or-string: true
+                                      type: object
+                                  type: object
+                                selector:
+                                  properties:
+                                    matchExpressions:
+                                      items:
+                                        properties:
+                                          key:
+                                            type: string
+                                          operator:
+                                            type: string
+                                          values:
+                                            items:
+                                              type: string
+                                            type: array
+                                        required:
+                                        - key
+                                        - operator
+                                        type: object
+                                      type: array
+                                    matchLabels:
+                                      additionalProperties:
+                                        type: string
+                                      type: object
+                                  type: object
+                                storageClassName:
+                                  type: string
+                                volumeMode:
+                                  type: string
+                                volumeName:
+                                  type: string
+                              type: object
+                          required:
+                          - spec
+                          type: object
+                      type: object
+                    fc:
+                      properties:
+                        fsType:
+                          type: string
+                        lun:
+                          format: int32
+                          type: integer
+                        readOnly:
+                          type: boolean
+                        targetWWNs:
+                          items:
+                            type: string
+                          type: array
+                        wwids:
+                          items:
+                            type: string
+                          type: array
+                      type: object
+                    flexVolume:
+                      properties:
+                        driver:
+                          type: string
+                        fsType:
+                          type: string
+                        options:
+                          additionalProperties:
+                            type: string
+                          type: object
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                      required:
+                      - driver
+                      type: object
+                    flocker:
+                      properties:
+                        datasetName:
+                          type: string
+                        datasetUUID:
+                          type: string
+                      type: object
+                    gcePersistentDisk:
+                      properties:
+                        fsType:
+                          type: string
+                        partition:
+                          format: int32
+                          type: integer
+                        pdName:
+                          type: string
+                        readOnly:
+                          type: boolean
+                      required:
+                      - pdName
+                      type: object
+                    gitRepo:
+                      properties:
+                        directory:
+                          type: string
+                        repository:
+                          type: string
+                        revision:
+                          type: string
+                      required:
+                      - repository
+                      type: object
+                    glusterfs:
+                      properties:
+                        endpoints:
+                          type: string
+                        path:
+                          type: string
+                        readOnly:
+                          type: boolean
+                      required:
+                      - endpoints
+                      - path
+                      type: object
+                    hostPath:
+                      properties:
+                        path:
+                          type: string
+                        type:
+                          type: string
+                      required:
+                      - path
+                      type: object
+                    iscsi:
+                      properties:
+                        chapAuthDiscovery:
+                          type: boolean
+                        chapAuthSession:
+                          type: boolean
+                        fsType:
+                          type: string
+                        initiatorName:
+                          type: string
+                        iqn:
+                          type: string
+                        iscsiInterface:
+                          type: string
+                        lun:
+                          format: int32
+                          type: integer
+                        portals:
+                          items:
+                            type: string
+                          type: array
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        targetPortal:
+                          type: string
+                      required:
+                      - iqn
+                      - lun
+                      - targetPortal
+                      type: object
+                    name:
+                      type: string
+                    nfs:
+                      properties:
+                        path:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        server:
+                          type: string
+                      required:
+                      - path
+                      - server
+                      type: object
+                    persistentVolumeClaim:
+                      properties:
+                        claimName:
+                          type: string
+                        readOnly:
+                          type: boolean
+                      required:
+                      - claimName
+                      type: object
+                    photonPersistentDisk:
+                      properties:
+                        fsType:
+                          type: string
+                        pdID:
+                          type: string
+                      required:
+                      - pdID
+                      type: object
+                    portworxVolume:
+                      properties:
+                        fsType:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        volumeID:
+                          type: string
+                      required:
+                      - volumeID
+                      type: object
+                    projected:
+                      properties:
+                        defaultMode:
+                          format: int32
+                          type: integer
+                        sources:
+                          items:
+                            properties:
+                              configMap:
+                                properties:
+                                  items:
+                                    items:
+                                      properties:
+                                        key:
+                                          type: string
+                                        mode:
+                                          format: int32
+                                          type: integer
+                                        path:
+                                          type: string
+                                      required:
+                                      - key
+                                      - path
+                                      type: object
+                                    type: array
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                type: object
+                              downwardAPI:
+                                properties:
+                                  items:
+                                    items:
+                                      properties:
+                                        fieldRef:
+                                          properties:
+                                            apiVersion:
+                                              type: string
+                                            fieldPath:
+                                              type: string
+                                          required:
+                                          - fieldPath
+                                          type: object
+                                        mode:
+                                          format: int32
+                                          type: integer
+                                        path:
+                                          type: string
+                                        resourceFieldRef:
+                                          properties:
+                                            containerName:
+                                              type: string
+                                            divisor:
+                                              anyOf:
+                                              - type: integer
+                                              - type: string
+                                              pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                                              x-kubernetes-int-or-string: true
+                                            resource:
+                                              type: string
+                                          required:
+                                          - resource
+                                          type: object
+                                      required:
+                                      - path
+                                      type: object
+                                    type: array
+                                type: object
+                              secret:
+                                properties:
+                                  items:
+                                    items:
+                                      properties:
+                                        key:
+                                          type: string
+                                        mode:
+                                          format: int32
+                                          type: integer
+                                        path:
+                                          type: string
+                                      required:
+                                      - key
+                                      - path
+                                      type: object
+                                    type: array
+                                  name:
+                                    type: string
+                                  optional:
+                                    type: boolean
+                                type: object
+                              serviceAccountToken:
+                                properties:
+                                  audience:
+                                    type: string
+                                  expirationSeconds:
+                                    format: int64
+                                    type: integer
+                                  path:
+                                    type: string
+                                required:
+                                - path
+                                type: object
+                            type: object
+                          type: array
+                      type: object
+                    quobyte:
+                      properties:
+                        group:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        registry:
+                          type: string
+                        tenant:
+                          type: string
+                        user:
+                          type: string
+                        volume:
+                          type: string
+                      required:
+                      - registry
+                      - volume
+                      type: object
+                    rbd:
+                      properties:
+                        fsType:
+                          type: string
+                        image:
+                          type: string
+                        keyring:
+                          type: string
+                        monitors:
+                          items:
+                            type: string
+                          type: array
+                        pool:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        user:
+                          type: string
+                      required:
+                      - image
+                      - monitors
+                      type: object
+                    scaleIO:
+                      properties:
+                        fsType:
+                          type: string
+                        gateway:
+                          type: string
+                        protectionDomain:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        sslEnabled:
+                          type: boolean
+                        storageMode:
+                          type: string
+                        storagePool:
+                          type: string
+                        system:
+                          type: string
+                        volumeName:
+                          type: string
+                      required:
+                      - gateway
+                      - secretRef
+                      - system
+                      type: object
+                    secret:
+                      properties:
+                        defaultMode:
+                          format: int32
+                          type: integer
+                        items:
+                          items:
+                            properties:
+                              key:
+                                type: string
+                              mode:
+                                format: int32
+                                type: integer
+                              path:
+                                type: string
+                            required:
+                            - key
+                            - path
+                            type: object
+                          type: array
+                        optional:
+                          type: boolean
+                        secretName:
+                          type: string
+                      type: object
+                    storageos:
+                      properties:
+                        fsType:
+                          type: string
+                        readOnly:
+                          type: boolean
+                        secretRef:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        volumeName:
+                          type: string
+                        volumeNamespace:
+                          type: string
+                      type: object
+                    vsphereVolume:
+                      properties:
+                        fsType:
+                          type: string
+                        storagePolicyID:
+                          type: string
+                        storagePolicyName:
+                          type: string
+                        volumePath:
+                          type: string
+                      required:
+                      - volumePath
+                      type: object
+                  required:
+                  - name
+                  type: object
+                type: array
+              zookeeper:
+                properties:
+                  spec:
+                    format: byte
+                    type: string
+                  type:
+                    type: string
+                required:
+                - spec
+                - type
+                type: object
+            required:
+            - common.runtime.properties
+            - commonConfigMountPath
+            - nodes
+            - startScript
+            type: object
+          status:
+            properties:
+              configMaps:
+                items:
+                  type: string
+                type: array
+              deployments:
+                items:
+                  type: string
+                type: array
+              druidNodeStatus:
+                properties:
+                  druidNode:
+                    type: string
+                  druidNodeConditionStatus:
+                    type: string
+                  druidNodeConditionType:
+                    type: string
+                  reason:
+                    type: string
+                type: object
+              hpAutoscalers:
+                items:
+                  type: string
+                type: array
+              ingress:
+                items:
+                  type: string
+                type: array
+              persistentVolumeClaims:
+                items:
+                  type: string
+                type: array
+              podDisruptionBudgets:
+                items:
+                  type: string
+                type: array
+              pods:
+                items:
+                  type: string
+                type: array
+              services:
+                items:
+                  type: string
+                type: array
+              statefulSets:
+                items:
+                  type: string
+                type: array
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/druid-operator/templates/deployment.yaml b/kubernetes/helm_charts/druid-operator/templates/deployment.yaml
new file mode 100644
index 0000000000..f4bdc5ae8d
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/deployment.yaml
@@ -0,0 +1,59 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "druid-operator.fullname" . }}
+  labels:
+    {{- include "druid-operator.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      {{- include "druid-operator.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "druid-operator.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "druid-operator.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          command:
+          - /manager
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            {{- range $key, $value :=  .Values.env }}
+            - name: {{ $key }}
+              value: {{ tpl $value $ | quote }}
+            {{- end }}
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
diff --git a/kubernetes/helm_charts/druid-operator/templates/role.yaml b/kubernetes/helm_charts/druid-operator/templates/role.yaml
new file mode 100644
index 0000000000..b12ed0d85e
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/role.yaml
@@ -0,0 +1,95 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if .Values.env.WATCH_NAMESPACE }}
+kind: Role
+{{- else }}
+kind: ClusterRole
+{{- end }}
+metadata:
+{{- if .Values.env.WATCH_NAMESPACE }}
+  namespace: {{ .Values.env.WATCH_NAMESPACE }}
+{{- end }}
+  name: {{ template "druid-operator.fullname" . }}
+  labels:
+    {{- include "druid-operator.labels" . | nindent 4 }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - configmaps
+      - services
+      - persistentvolumeclaims
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+  - apiGroups:
+      - apps
+    resources:
+      - statefulsets
+      - deployments
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - delete
+  - apiGroups:
+      - autoscaling
+    resources:
+      - horizontalpodautoscalers
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+      - patch
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - ingresses
+    verbs:
+      - list
+      - watch
+      - create
+      - update
+  - apiGroups:
+      - druid.apache.org
+    resources:
+      - druids
+    verbs:
+      - get
+      - list
+      - watch
+      - create
+      - update
+      - patch
+      - delete
+  - apiGroups:
+      - druid.apache.org
+    resources:
+      - druids/status
+    verbs:
+      - get
+      - update
+      - patch
+{{- end }}
diff --git a/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml b/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml
new file mode 100644
index 0000000000..2587fe6047
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/role_binding.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+{{- if .Values.env.WATCH_NAMESPACE }}
+kind: RoleBinding
+{{- else }}
+kind: ClusterRoleBinding
+{{- end }}
+metadata:
+{{- if .Values.env.WATCH_NAMESPACE }}
+  namespace: {{ .Values.env.WATCH_NAMESPACE }}
+{{- end }}
+  name: {{ template "druid-operator.fullname" . }}
+  labels:
+    {{- include "druid-operator.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "druid-operator.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: {{ if .Values.env.WATCH_NAMESPACE }} Role {{ else }} ClusterRole {{ end }}
+  name: {{ template "druid-operator.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml b/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml
new file mode 100644
index 0000000000..d1af85fe91
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/templates/serviceaccount.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: {{ include "druid-operator.serviceAccountName" . }}
+  labels:
+    {{- include "druid-operator.labels" . | nindent 4 }}
+  {{- with .Values.serviceAccount.annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/druid-operator/values.yaml b/kubernetes/helm_charts/druid-operator/values.yaml
new file mode 100644
index 0000000000..75ab618eca
--- /dev/null
+++ b/kubernetes/helm_charts/druid-operator/values.yaml
@@ -0,0 +1,63 @@
+# Default values for druid-operator.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+env:
+  DENY_LIST: "default,kube-system" # Comma-separated list of namespaces to ignore
+  RECONCILE_WAIT: "10s"            # Reconciliation delay
+  WATCH_NAMESPACE: ""              # Namespace to watch or empty string to watch all namespaces
+
+replicaCount: 1
+
+image:
+  repository: druidio/druid-operator
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+rbac:
+  create: true
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations: {}
+
+podSecurityContext: {}
+  # fsGroup: 2000
+
+securityContext: {}
+  # capabilities:
+  #   drop:
+  #   - ALL
+  # readOnlyRootFilesystem: true
+  # runAsNonRoot: true
+  # runAsUser: 1000
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
diff --git a/kubernetes/helm_charts/kafka/Chart.yaml b/kubernetes/helm_charts/kafka/Chart.yaml
new file mode 100644
index 0000000000..5d110918bd
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/Chart.yaml
@@ -0,0 +1,32 @@
+annotations:
+  category: Infrastructure
+apiVersion: v2
+appVersion: 2.8.0
+dependencies:
+  - name: common
+    repository: https://charts.bitnami.com/bitnami
+    tags:
+      - bitnami-common
+    version: 1.x.x
+  - condition: zookeeper.enabled
+    name: zookeeper
+    repository: https://charts.bitnami.com/bitnami
+    version: 7.x.x
+description: Apache Kafka is a distributed streaming platform.
+engine: gotpl
+home: https://github.com/bitnami/charts/tree/master/bitnami/kafka
+icon: https://bitnami.com/assets/stacks/kafka/img/kafka-stack-220x234.png
+keywords:
+  - kafka
+  - zookeeper
+  - streaming
+  - producer
+  - consumer
+maintainers:
+  - email: containers@bitnami.com
+    name: Bitnami
+name: kafka
+sources:
+  - https://github.com/bitnami/bitnami-docker-kafka
+  - https://kafka.apache.org/
+version: 14.0.0
\ No newline at end of file
diff --git a/kubernetes/helm_charts/kafka/templates/NOTES.txt b/kubernetes/helm_charts/kafka/templates/NOTES.txt
new file mode 100644
index 0000000000..ed93cdf77d
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/NOTES.txt
@@ -0,0 +1,256 @@
+CHART NAME: {{ .Chart.Name }}
+CHART VERSION: {{ .Chart.Version }}
+APP VERSION: {{ .Chart.AppVersion }}
+
+{{- if .Values.diagnosticMode.enabled }}
+The chart has been deployed in diagnostic mode. All probes have been disabled and the command has been overwritten with:
+
+  command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 4 }}
+  args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 4 }}
+
+Get the list of pods by executing:
+
+  kubectl get pods --namespace {{ .Release.Namespace }} -l app.kubernetes.io/instance={{ .Release.Name }}
+
+Access the pod you want to debug by executing
+
+  kubectl exec --namespace {{ .Release.Namespace }} -ti <NAME OF THE POD> -- bash
+
+In order to replicate the container startup scripts execute this command:
+
+    /opt/bitnami/scripts/kafka/entrypoint.sh /opt/bitnami/scripts/kafka/run.sh
+
+{{- else }}
+
+{{- $replicaCount := int .Values.replicaCount -}}
+{{- $releaseNamespace := .Release.Namespace -}}
+{{- $clusterDomain := .Values.clusterDomain -}}
+{{- $fullname := include "kafka.fullname" . -}}
+{{- $clientProtocol := include "kafka.listenerType" (dict "protocol" .Values.auth.clientProtocol) -}}
+{{- $saslMechanisms := coalesce .Values.auth.sasl.mechanisms .Values.auth.saslMechanisms -}}
+{{- $tlsEndpointIdentificationAlgorithm := default "" (coalesce .Values.auth.tls.endpointIdentificationAlgorithm .Values.auth.tlsEndpointIdentificationAlgorithm) -}}
+{{- $tlsPassword := coalesce .Values.auth.tls.password .Values.auth.jksPassword -}}
+{{- $servicePort := int .Values.service.port -}}
+{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs -}}
+{{- if and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer") }}
+
+###############################################################################
+### ERROR: You enabled external access to Kafka brokers without specifying  ###
+###   the array of load balancer IPs for Kafka brokers.                     ###
+###############################################################################
+
+This deployment will be incomplete until you configure the array of load balancer
+IPs for Kafka brokers. To complete your deployment follow the steps below:
+
+1. Wait for the load balancer IPs (it may take a few minutes for them to be available):
+
+    kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -w
+
+2. Obtain the load balancer IPs and upgrade your chart:
+
+    {{- range $i, $e := until $replicaCount }}
+    LOAD_BALANCER_IP_{{ add $i 1 }}="$(kubectl get svc --namespace {{ $releaseNamespace }} {{ $fullname }}-{{ $i }}-external -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
+    {{- end }}
+
+3. Upgrade you chart:
+
+    helm upgrade --namespace {{ .Release.Namespace }} {{ .Release.Name }} bitnami/{{ .Chart.Name }} \
+      --set replicaCount={{ $replicaCount }} \
+      --set externalAccess.enabled=true \
+      {{- range $i, $e := until $replicaCount }}
+      --set externalAccess.service.loadBalancerIPs[{{ $i }}]=$LOAD_BALANCER_IP_{{ add $i 1 }} \
+      {{- end }}
+      --set externalAccess.service.type=LoadBalancer
+
+{{- else }}
+
+{{- if and (or (eq .Values.service.type "LoadBalancer") .Values.externalAccess.enabled) (eq $clientProtocol "PLAINTEXT") }}
+---------------------------------------------------------------------------------------------
+ WARNING
+
+    By specifying "serviceType=LoadBalancer" and not configuring the authentication
+    you have most likely exposed the Kafka service externally without any
+    authentication mechanism.
+
+    For security reasons, we strongly suggest that you switch to "ClusterIP" or
+    "NodePort". As alternative, you can also configure the Kafka authentication.
+
+---------------------------------------------------------------------------------------------
+{{- end }}
+
+** Please be patient while the chart is being deployed **
+
+Kafka can be accessed by consumers via port {{ $servicePort }} on the following DNS name from within your cluster:
+
+    {{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}
+
+Each Kafka broker can be accessed by producers via port {{ $servicePort }} on the following DNS name(s) from within your cluster:
+
+{{- $brokerList := list }}
+{{- range $e, $i := until $replicaCount }}
+{{- $brokerList = append $brokerList (printf "%s-%d.%s-headless.%s.svc.%s:%d" $fullname $i $fullname $releaseNamespace $clusterDomain $servicePort) }}
+{{- end }}
+{{ join "\n" $brokerList | nindent 4 }}
+{{- if (include "kafka.client.saslAuthentication" .) }}
+
+You need to configure your Kafka client to access using SASL authentication. To do so, you need to create the 'kafka_jaas.conf' and 'client.properties' configuration files with the content below:
+
+    - kafka_jaas.conf:
+
+KafkaClient {
+{{- if $saslMechanisms | regexFind "scram" }}
+org.apache.kafka.common.security.scram.ScramLoginModule required
+{{- else }}
+org.apache.kafka.common.security.plain.PlainLoginModule required
+{{- end }}
+username="{{ index (coalesce .Values.auth.sasl.jaas.clientUsers .Values.auth.jaas.clientUsers) 0 }}"
+password="$(kubectl get secret {{ $fullname }}-jaas --namespace {{ $releaseNamespace }} -o jsonpath='{.data.client-passwords}' | base64 --decode | cut -d , -f 1)";
+};
+
+    - client.properties:
+
+security.protocol={{ $clientProtocol }}
+{{- if $saslMechanisms | regexFind "scram-sha-256" }}
+sasl.mechanism=SCRAM-SHA-256
+{{- else if $saslMechanisms | regexFind "scram-sha-512" }}
+sasl.mechanism=SCRAM-SHA-512
+{{- else }}
+sasl.mechanism=PLAIN
+{{- end }}
+{{- if eq $clientProtocol "SASL_SSL" }}
+ssl.truststore.type={{ upper .Values.auth.tls.type }}
+    {{- if eq .Values.auth.tls.type "jks" }}
+ssl.truststore.location=/tmp/kafka.truststore.jks
+        {{- if not (empty $tlsPassword) }}
+ssl.truststore.password={{ $tlsPassword }}
+        {{- end }}
+    {{- else if eq .Values.auth.tls.type "pem" }}
+ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
+... \
+-----END CERTIFICATE-----
+    {{- end }}
+    {{- if eq $tlsEndpointIdentificationAlgorithm "" }}
+ssl.endpoint.identification.algorithm=
+    {{- end }}
+{{- end }}
+
+{{- else if (include "kafka.client.tlsEncryption" .) }}
+
+You need to configure your Kafka client to access using TLS authentication. To do so, you need to create the 'client.properties' configuration file with the content below:
+
+security.protocol={{ $clientProtocol }}
+ssl.truststore.type={{ upper .Values.auth.tls.type }}
+{{- if eq .Values.auth.tls.type "jks" }}
+ssl.truststore.location=/tmp/kafka.truststore.{{ .Values.auth.tls.type }}
+    {{- if not (empty $tlsPassword) }}
+ssl.truststore.password={{ $tlsPassword }}
+    {{- end }}
+{{- else if eq .Values.auth.tls.type "pem" }}
+ssl.truststore.certificates=-----BEGIN CERTIFICATE----- \
+... \
+-----END CERTIFICATE-----
+{{- end }}
+{{- if eq .Values.auth.clientProtocol "mtls" }}
+ssl.keystore.type={{ upper .Values.auth.tls.type }}
+    {{- if eq .Values.auth.tls.type "jks" }}
+ssl.keystore.location=/tmp/client.keystore.jks
+        {{- if not (empty $tlsPassword) }}
+ssl.keystore.password={{ $tlsPassword }}
+        {{- end }}
+    {{- else if eq .Values.auth.tls.type "pem" }}
+ssl.keystore.certificate.chain=-----BEGIN CERTIFICATE----- \
+... \
+-----END CERTIFICATE-----
+ssl.keystore.key=-----BEGIN ENCRYPTED PRIVATE KEY----- \
+... \
+-----END ENCRYPTED PRIVATE KEY-----
+    {{- end }}
+{{- end }}
+{{- if eq $tlsEndpointIdentificationAlgorithm "" }}
+ssl.endpoint.identification.algorithm=
+{{- end }}
+
+{{- end }}
+
+To create a pod that you can use as a Kafka client run the following commands:
+
+    kubectl run {{ $fullname }}-client --restart='Never' --image {{ template "kafka.image" . }} --namespace {{ $releaseNamespace }} --command -- sleep infinity
+    {{- if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}
+    kubectl cp --namespace {{ $releaseNamespace }} /path/to/client.properties {{ $fullname }}-client:/tmp/client.properties
+    {{- end }}
+    {{- if (include "kafka.client.saslAuthentication" .) }}
+    kubectl cp --namespace {{ $releaseNamespace }} /path/to/kafka_jaas.conf {{ $fullname }}-client:/tmp/kafka_jaas.conf
+    {{- end }}
+    {{- if and (include "kafka.client.tlsEncryption" .) (eq .Values.auth.tls.type "jks") }}
+    kubectl cp --namespace {{ $releaseNamespace }} ./kafka.truststore.jks {{ $fullname }}-client:/tmp/kafka.truststore.jks
+    {{- if eq .Values.auth.clientProtocol "mtls" }}
+    kubectl cp --namespace {{ $releaseNamespace }} ./client.keystore.jks {{ $fullname }}-client:/tmp/client.keystore.jks
+    {{- end }}
+    {{- end }}
+    kubectl exec --tty -i {{ $fullname }}-client --namespace {{ $releaseNamespace }} -- bash
+    {{- if (include "kafka.client.saslAuthentication" .) }}
+    export KAFKA_OPTS="-Djava.security.auth.login.config=/tmp/kafka_jaas.conf"
+    {{- end }}
+
+    PRODUCER:
+        kafka-console-producer.sh \
+            {{ if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}--producer.config /tmp/client.properties \{{ end }}
+            --broker-list {{ join "," $brokerList }} \
+            --topic test
+
+    CONSUMER:
+        kafka-console-consumer.sh \
+            {{ if or (include "kafka.client.saslAuthentication" .) (include "kafka.client.tlsEncryption" .) }}--consumer.config /tmp/client.properties \{{ end }}
+            --bootstrap-server {{ $fullname }}.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ .Values.service.port }} \
+            --topic test \
+            --from-beginning
+
+{{- if .Values.externalAccess.enabled }}
+
+To connect to your Kafka server from outside the cluster, follow the instructions below:
+
+{{- if eq "NodePort" .Values.externalAccess.service.type }}
+{{- if .Values.externalAccess.service.domain }}
+
+    Kafka brokers domain: Use your provided hostname to reach Kafka brokers, {{ .Values.externalAccess.service.domain }}
+
+{{- else }}
+
+    Kafka brokers domain: You can get the external node IP from the Kafka configuration file with the following commands (Check the EXTERNAL listener)
+
+        1. Obtain the pod name:
+
+        kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka"
+
+        2. Obtain pod configuration:
+
+        kubectl exec -it KAFKA_POD -- cat /opt/bitnami/kafka/config/server.properties | grep advertised.listeners
+
+{{- end }}
+
+    Kafka brokers port: You will have a different node port for each Kafka broker. You can get the list of configured node ports using the command below:
+
+        echo "$(kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -o jsonpath='{.items[*].spec.ports[0].nodePort}' | tr ' ' '\n')"
+
+{{- else if contains "LoadBalancer" .Values.externalAccess.service.type }}
+
+  NOTE: It may take a few minutes for the LoadBalancer IPs to be available.
+        Watch the status with: 'kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -w'
+
+    Kafka Brokers domain: You will have a different external IP for each Kafka broker. You can get the list of external IPs using the command below:
+
+        echo "$(kubectl get svc --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ template "kafka.name" . }},app.kubernetes.io/instance={{ .Release.Name }},app.kubernetes.io/component=kafka,pod" -o jsonpath='{.items[*].status.loadBalancer.ingress[0].ip}' | tr ' ' '\n')"
+
+    Kafka Brokers port: {{ .Values.externalAccess.service.port }}
+
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{- include "common.warnings.rollingTag" .Values.image }}
+{{- include "common.warnings.rollingTag" .Values.externalAccess.autoDiscovery.image }}
+{{- include "common.warnings.rollingTag" .Values.metrics.kafka.image }}
+{{- include "common.warnings.rollingTag" .Values.metrics.jmx.image }}
+{{- include "common.warnings.rollingTag" .Values.volumePermissions.image }}
+{{- include "kafka.validateValues" . }}
diff --git a/kubernetes/helm_charts/kafka/templates/_helpers.tpl b/kubernetes/helm_charts/kafka/templates/_helpers.tpl
new file mode 100644
index 0000000000..2c732159e2
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/_helpers.tpl
@@ -0,0 +1,407 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kafka.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kafka.fullname" -}}
+{{- include "common.names.fullname" . -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified zookeeper name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "kafka.zookeeper.fullname" -}}
+{{- if .Values.zookeeper.fullnameOverride -}}
+{{- .Values.zookeeper.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default "zookeeper" .Values.zookeeper.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+ Create the name of the service account to use
+ */}}
+{{- define "kafka.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "kafka.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Kafka image name
+*/}}
+{{- define "kafka.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper image name (for the init container auto-discovery image)
+*/}}
+{{- define "kafka.externalAccess.autoDiscovery.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.externalAccess.autoDiscovery.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper image name (for the init container volume-permissions image)
+*/}}
+{{- define "kafka.volumePermissions.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.volumePermissions.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Create a default fully qualified Kafka exporter name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "kafka.metrics.kafka.fullname" -}}
+  {{- printf "%s-exporter" (include "kafka.fullname" .) | trunc 63 | trimSuffix "-" }}
+{{- end -}}
+
+{{/*
+ Create the name of the service account to use for Kafka exporer pods
+ */}}
+{{- define "kafka.metrics.kafka.serviceAccountName" -}}
+{{- if .Values.metrics.kafka.serviceAccount.create -}}
+    {{ default (include "kafka.metrics.kafka.fullname" .) .Values.metrics.kafka.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.metrics.kafka.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the proper Kafka exporter image name
+*/}}
+{{- define "kafka.metrics.kafka.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.metrics.kafka.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper JMX exporter image name
+*/}}
+{{- define "kafka.metrics.jmx.image" -}}
+{{ include "common.images.image" (dict "imageRoot" .Values.metrics.jmx.image "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper Docker Image Registry Secret Names
+*/}}
+{{- define "kafka.imagePullSecrets" -}}
+{{ include "common.images.pullSecrets" (dict "images" (list .Values.image .Values.externalAccess.autoDiscovery.image .Values.volumePermissions.image .Values.metrics.kafka.image .Values.metrics.jmx.image) "global" .Values.global) }}
+{{- end -}}
+
+{{/*
+Return the proper Storage Class
+*/}}
+{{- define "kafka.storageClass" -}}
+{{/*
+Helm 2.11 supports the assignment of a value to a variable defined in a different scope,
+but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic.
+*/}}
+{{- if .Values.global -}}
+    {{- if .Values.global.storageClass -}}
+        {{- if (eq "-" .Values.global.storageClass) -}}
+            {{- printf "storageClassName: \"\"" -}}
+        {{- else }}
+            {{- printf "storageClassName: %s" .Values.global.storageClass -}}
+        {{- end -}}
+    {{- else -}}
+        {{- if .Values.persistence.storageClass -}}
+              {{- if (eq "-" .Values.persistence.storageClass) -}}
+                  {{- printf "storageClassName: \"\"" -}}
+              {{- else }}
+                  {{- printf "storageClassName: %s" .Values.persistence.storageClass -}}
+              {{- end -}}
+        {{- end -}}
+    {{- end -}}
+{{- else -}}
+    {{- if .Values.persistence.storageClass -}}
+        {{- if (eq "-" .Values.persistence.storageClass) -}}
+            {{- printf "storageClassName: \"\"" -}}
+        {{- else }}
+            {{- printf "storageClassName: %s" .Values.persistence.storageClass -}}
+        {{- end -}}
+    {{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if authentication via SASL should be configured for client communications
+*/}}
+{{- define "kafka.client.saslAuthentication" -}}
+{{- $saslProtocols := list "sasl" "sasl_tls" -}}
+{{- if has .Values.auth.clientProtocol $saslProtocols -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if authentication via SASL should be configured for inter-broker communications
+*/}}
+{{- define "kafka.interBroker.saslAuthentication" -}}
+{{- $saslProtocols := list "sasl" "sasl_tls" -}}
+{{- if has .Values.auth.interBrokerProtocol $saslProtocols -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if encryption via TLS for client connections should be configured
+*/}}
+{{- define "kafka.client.tlsEncryption" -}}
+{{- $tlsProtocols := list "tls" "mtls" "sasl_tls" -}}
+{{- if (has .Values.auth.clientProtocol $tlsProtocols) -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if encryption via TLS for inter broker communication connections should be configured
+*/}}
+{{- define "kafka.interBroker.tlsEncryption" -}}
+{{- $tlsProtocols := list "tls" "mtls" "sasl_tls" -}}
+{{- if (has .Values.auth.interBrokerProtocol $tlsProtocols) -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if encryption via TLS should be configured
+*/}}
+{{- define "kafka.tlsEncryption" -}}
+{{- if or (include "kafka.client.tlsEncryption" .) (include "kafka.interBroker.tlsEncryption" .) -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the type of listener
+Usage:
+{{ include "kafka.listenerType" ( dict "protocol" .Values.path.to.the.Value ) }}
+*/}}
+{{- define "kafka.listenerType" -}}
+{{- if eq .protocol "plaintext" -}}
+PLAINTEXT
+{{- else if or (eq .protocol "tls") (eq .protocol "mtls") -}}
+SSL
+{{- else if eq .protocol "sasl_tls" -}}
+SASL_SSL
+{{- else if eq .protocol "sasl" -}}
+SASL_PLAINTEXT
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the Kafka JAAS credentials secret
+*/}}
+{{- define "kafka.jaasSecretName" -}}
+{{- $secretName := coalesce .Values.auth.sasl.jaas.existingSecret .Values.auth.jaas.existingSecret -}}
+{{- if $secretName -}}
+    {{- printf "%s" (tpl $secretName $) -}}
+{{- else -}}
+    {{- printf "%s-jaas" (include "kafka.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a JAAS credentials secret object should be created
+*/}}
+{{- define "kafka.createJaasSecret" -}}
+{{- $secretName := coalesce .Values.auth.sasl.jaas.existingSecret .Values.auth.jaas.existingSecret -}}
+{{- if and (or (include "kafka.client.saslAuthentication" .) (include "kafka.interBroker.saslAuthentication" .) (and .Values.zookeeper.auth.enabled .Values.auth.jaas.zookeeperUser)) (empty $secretName) -}}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a TLS credentials secret object should be created
+*/}}
+{{- define "kafka.createTlsSecret" -}}
+{{- $secretName := coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret -}}
+{{- if and (include "kafka.tlsEncryption" .) (empty .Values.auth.tls.existingSecrets) (empty $secretName) (eq .Values.auth.tls.type "jks") (.Files.Glob "files/tls/*.jks") }}
+    {{- true -}}
+{{- else if and (include "kafka.tlsEncryption" .) (empty .Values.auth.tls.existingSecrets) (empty $secretName) (eq .Values.auth.tls.type "pem") (or (.Files.Glob "files/tls/*.{crt,pem}") .Values.auth.tls.autoGenerated) }}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the Kafka configuration configmap
+*/}}
+{{- define "kafka.configmapName" -}}
+{{- if .Values.existingConfigmap -}}
+    {{- printf "%s" (tpl .Values.existingConfigmap $) -}}
+{{- else -}}
+    {{- printf "%s-configuration" (include "kafka.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a configmap object should be created
+*/}}
+{{- define "kafka.createConfigmap" -}}
+{{- if and .Values.config (not .Values.existingConfigmap) }}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the Kafka log4j ConfigMap name.
+*/}}
+{{- define "kafka.log4j.configMapName" -}}
+{{- if .Values.existingLog4jConfigMap -}}
+    {{- printf "%s" (tpl .Values.existingLog4jConfigMap $) -}}
+{{- else -}}
+    {{- printf "%s-log4j-configuration" (include "kafka.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a log4j ConfigMap object should be created.
+*/}}
+{{- define "kafka.log4j.createConfigMap" -}}
+{{- if and .Values.log4j (not .Values.existingLog4jConfigMap) }}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the SASL mechanism to use for the Kafka exporter to access Kafka
+The exporter uses a different nomenclature so we need to do this hack
+*/}}
+{{- define "kafka.metrics.kafka.saslMechanism" -}}
+{{- $saslMechanisms := coalesce .Values.auth.sasl.mechanisms .Values.auth.saslMechanisms }}
+{{- if contains "scram-sha-512" $saslMechanisms }}
+    {{- printf "scram-sha512" -}}
+{{- else if contains "scram-sha-256" $saslMechanisms }}
+    {{- printf "scram-sha256" -}}
+{{- else -}}
+    {{- printf "plain" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return the Kafka configuration configmap
+*/}}
+{{- define "kafka.metrics.jmx.configmapName" -}}
+{{- if .Values.metrics.jmx.existingConfigmap -}}
+    {{- printf "%s" (tpl .Values.metrics.jmx.existingConfigmap $) -}}
+{{- else -}}
+    {{- printf "%s-jmx-configuration" (include "kafka.fullname" .) -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Return true if a configmap object should be created
+*/}}
+{{- define "kafka.metrics.jmx.createConfigmap" -}}
+{{- if and .Values.metrics.jmx.enabled .Values.metrics.jmx.config (not .Values.metrics.jmx.existingConfigmap) }}
+    {{- true -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Compile all warnings into a single message, and call fail.
+*/}}
+{{- define "kafka.validateValues" -}}
+{{- $messages := list -}}
+{{- $messages := append $messages (include "kafka.validateValues.authProtocols" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.nodePortListLength" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.externalAccessServiceType" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.externalAccessAutoDiscoveryRBAC" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.saslMechanisms" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.tlsSecrets" .) -}}
+{{- $messages := append $messages (include "kafka.validateValues.tlsSecrets.length" .) -}}
+{{- $messages := without $messages "" -}}
+{{- $message := join "\n" $messages -}}
+
+{{- if $message -}}
+{{-   printf "\nVALUES VALIDATION:\n%s" $message | fail -}}
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - Authentication protocols for Kafka */}}
+{{- define "kafka.validateValues.authProtocols" -}}
+{{- $authProtocols := list "plaintext" "tls" "mtls" "sasl" "sasl_tls" -}}
+{{- if or (not (has .Values.auth.clientProtocol $authProtocols)) (not (has .Values.auth.interBrokerProtocol $authProtocols)) -}}
+kafka: auth.clientProtocol auth.interBrokerProtocol
+    Available authentication protocols are "plaintext", "tls", "mtls", "sasl" and "sasl_tls"
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - number of replicas must be the same than NodePort list */}}
+{{- define "kafka.validateValues.nodePortListLength" -}}
+{{- $replicaCount := int .Values.replicaCount }}
+{{- $nodePortListLength := len .Values.externalAccess.service.nodePorts }}
+{{- if and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $nodePortListLength )) (eq .Values.externalAccess.service.type "NodePort") -}}
+kafka: .Values.externalAccess.service.nodePorts
+    Number of replicas and nodePort array length must be the same. Currently: replicaCount = {{ $replicaCount }} and nodePorts = {{ $nodePortListLength }}
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - service type for external access */}}
+{{- define "kafka.validateValues.externalAccessServiceType" -}}
+{{- if and (not (eq .Values.externalAccess.service.type "NodePort")) (not (eq .Values.externalAccess.service.type "LoadBalancer")) -}}
+kafka: externalAccess.service.type
+    Available service type for external access are NodePort or LoadBalancer.
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - RBAC should be enabled when autoDiscovery is enabled */}}
+{{- define "kafka.validateValues.externalAccessAutoDiscoveryRBAC" -}}
+{{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled (not .Values.rbac.create )}}
+kafka: rbac.create
+    By specifying "externalAccess.enabled=true" and "externalAccess.autoDiscovery.enabled=true"
+    an initContainer will be used to auto-detect the external IPs/ports by querying the
+    K8s API. Please note this initContainer requires specific RBAC resources. You can create them
+    by specifying "--set rbac.create=true".
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - SASL mechanisms must be provided when using SASL */}}
+{{- define "kafka.validateValues.saslMechanisms" -}}
+{{- if and (or (.Values.auth.clientProtocol | regexFind "sasl") (.Values.auth.interBrokerProtocol | regexFind "sasl") (and .Values.zookeeper.auth.enabled .Values.auth.jaas.zookeeperUser)) (not .Values.auth.saslMechanisms) }}
+kafka: auth.saslMechanisms
+    The SASL mechanisms are required when either auth.clientProtocol or auth.interBrokerProtocol use SASL or Zookeeper user is provided.
+{{- end }}
+{{- if not (contains .Values.auth.saslInterBrokerMechanism .Values.auth.saslMechanisms) }}
+kafka: auth.saslMechanisms
+    auth.saslInterBrokerMechanism must be provided and it should be one of the specified mechanisms at auth.saslMechanisms
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - Secrets containing TLS certs must be provided when TLS authentication is enabled */}}
+{{- define "kafka.validateValues.tlsSecrets" -}}
+{{- $secretName := coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret -}}
+{{- if and (include "kafka.tlsEncryption" .) (eq .Values.auth.tls.type "jks") (empty .Values.auth.tls.existingSecrets) (empty $secretName) (not (.Files.Glob "files/tls/*.jks}")) }}
+kafka: auth.tls.existingSecret
+    A secret containing the Kafka JKS keystores and truststore is required
+    when TLS encryption in enabled and TLS format is "JKS"
+{{- else if and (include "kafka.tlsEncryption" .) (eq .Values.auth.tls.type "pem") (empty .Values.auth.tls.existingSecrets) (empty $secretName) (not (.Files.Glob "files/tls/*.{crt,pem}")) (not .Values.auth.tls.autoGenerated) }}
+kafka: auth.tls.existingSecret
+    A secret containing the Kafka TLS certificates and keys is required
+    when TLS encryption in enabled and TLS format is "PEM"
+{{- end -}}
+{{- end -}}
+
+{{/* Validate values of Kafka - The number of secrets containing TLS certs should be equal to the number of replicas */}}
+{{- define "kafka.validateValues.tlsSecrets.length" -}}
+{{- $replicaCount := int .Values.replicaCount }}
+{{- if and (include "kafka.tlsEncryption" .) (not (empty .Values.auth.tls.existingSecrets)) }}
+{{- $existingSecretsLength := len .Values.auth.tls.existingSecrets }}
+{{- if ne $replicaCount $existingSecretsLength }}
+kafka: .Values.auth.tls.existingSecrets
+    Number of replicas and existingSecrets array length must be the same. Currently: replicaCount = {{ $replicaCount }} and existingSecrets = {{ $existingSecretsLength }}
+{{- end -}}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/kafka/templates/configmap.yaml b/kubernetes/helm_charts/kafka/templates/configmap.yaml
new file mode 100644
index 0000000000..717c0c7d83
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/configmap.yaml
@@ -0,0 +1,16 @@
+{{- if (include "kafka.createConfigmap" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "kafka.fullname" . }}-configuration
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  server.properties: |-
+    {{ .Values.config | nindent 4 }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/kafka/templates/extra-list.yaml b/kubernetes/helm_charts/kafka/templates/extra-list.yaml
new file mode 100644
index 0000000000..9ac65f9e16
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/extra-list.yaml
@@ -0,0 +1,4 @@
+{{- range .Values.extraDeploy }}
+---
+{{ include "common.tplvalues.render" (dict "value" . "context" $) }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/jaas-secret.yaml b/kubernetes/helm_charts/kafka/templates/jaas-secret.yaml
new file mode 100644
index 0000000000..7fa3a921c1
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/jaas-secret.yaml
@@ -0,0 +1,37 @@
+{{- if (include "kafka.createJaasSecret" .) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ template "kafka.fullname" . }}-jaas
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+type: Opaque
+data:
+  {{- if (include "kafka.client.saslAuthentication" .) }}
+  {{- $clientUsers := coalesce .Values.auth.sasl.jaas.clientUsers .Values.auth.jaas.clientUsers }}
+  {{- $clientPasswords := coalesce .Values.auth.sasl.jaas.clientPasswords .Values.auth.jaas.clientPasswords }}
+  {{- if $clientPasswords }}
+  client-passwords: {{ join "," $clientPasswords | b64enc | quote }}
+  {{- else }}
+  {{- $passwords := list }}
+  {{- range $clientUsers }}
+  {{- $passwords = append $passwords (randAlphaNum 10) }}
+  {{- end }}
+  client-passwords: {{ join "," $passwords | b64enc | quote }}
+  {{- end }}
+  {{- end }}
+  {{- $zookeeperUser := coalesce .Values.auth.sasl.jaas.zookeeperUser .Values.auth.jaas.zookeeperUser }}
+  {{- if and .Values.zookeeper.auth.enabled $zookeeperUser }}
+  {{- $zookeeperPassword := coalesce .Values.auth.sasl.jaas.zookeeperPassword .Values.auth.jaas.zookeeperPassword }}
+  zookeeper-password: {{ default (randAlphaNum 10) $zookeeperPassword | b64enc | quote }}
+  {{- end }}
+  {{- if (include "kafka.interBroker.saslAuthentication" .) }}
+  {{- $interBrokerPassword := coalesce .Values.auth.sasl.jaas.interBrokerPassword .Values.auth.jaas.interBrokerPassword }}
+  inter-broker-password: {{ default (randAlphaNum 10) $interBrokerPassword | b64enc | quote }}
+  {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/jmx-configmap.yaml b/kubernetes/helm_charts/kafka/templates/jmx-configmap.yaml
new file mode 100644
index 0000000000..9a24a942a5
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/jmx-configmap.yaml
@@ -0,0 +1,59 @@
+{{- if (include "kafka.metrics.jmx.createConfigmap" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "kafka.fullname" . }}-jmx-configuration
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  jmx-kafka-prometheus.yml: |-
+    {{- include "common.tplvalues.render" ( dict "value" .Values.metrics.jmx.config "context" $ ) | nindent 4 }}
+    rules:
+      - pattern: kafka.controller<type=(ControllerChannelManager), name=(QueueSize), broker-id=(\d+)><>(Value)
+        name: kafka_controller_$1_$2_$4
+        labels:
+          broker_id: "$3"
+      - pattern: kafka.controller<type=(ControllerChannelManager), name=(TotalQueueSize)><>(Value)
+        name: kafka_controller_$1_$2_$3
+      - pattern: kafka.controller<type=(KafkaController), name=(.+)><>(Value)
+        name: kafka_controller_$1_$2_$3
+      - pattern: kafka.controller<type=(ControllerStats), name=(.+)><>(Count)
+        name: kafka_controller_$1_$2_$3
+      - pattern: kafka.server<type=(ReplicaFetcherManager), name=(.+), clientId=(.+)><>(Value)
+        name: kafka_server_$1_$2_$4
+        labels:
+          client_id: "$3"
+      - pattern : kafka.network<type=(Processor), name=(IdlePercent), networkProcessor=(.+)><>(Value)
+        name: kafka_network_$1_$2_$4
+        labels:
+          network_processor: $3
+      - pattern : kafka.network<type=(RequestMetrics), name=(RequestsPerSec), request=(.+)><>(Count)
+        name: kafka_network_$1_$2_$4
+        labels:
+          request: $3
+      - pattern: kafka.server<type=(.+), name=(.+), topic=(.+)><>(Count|OneMinuteRate)
+        name: kafka_server_$1_$2_$4
+        labels:
+          topic: $3
+      - pattern: kafka.server<type=(DelayedOperationPurgatory), name=(.+), delayedOperation=(.+)><>(Value)
+        name: kafka_server_$1_$2_$3_$4
+      - pattern: kafka.server<type=(.+), name=(.+)><>(Count|Value|OneMinuteRate)
+        name: kafka_server_$1_total_$2_$3
+      - pattern: kafka.server<type=(.+)><>(queue-size)
+        name: kafka_server_$1_$2
+      - pattern: java.lang<type=(.+), name=(.+)><(.+)>(\w+)
+        name: java_lang_$1_$4_$3_$2
+      - pattern: java.lang<type=(.+), name=(.+)><>(\w+)
+        name: java_lang_$1_$3_$2
+      - pattern : java.lang<type=(.*)>
+      - pattern: kafka.log<type=(.+), name=(.+), topic=(.+), partition=(.+)><>Value
+        name: kafka_log_$1_$2
+        labels:
+          topic: $3
+          partition: $4
+{{- end -}}
diff --git a/kubernetes/helm_charts/kafka/templates/jmx-metrics-svc.yaml b/kubernetes/helm_charts/kafka/templates/jmx-metrics-svc.yaml
new file mode 100644
index 0000000000..9f70d668f8
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/jmx-metrics-svc.yaml
@@ -0,0 +1,45 @@
+{{- if .Values.metrics.jmx.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kafka.fullname" . }}-jmx-metrics
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if or .Values.metrics.jmx.service.annotations .Values.commonAnnotations }}
+  annotations:
+    {{- if .Values.metrics.jmx.service.annotations }}
+    {{ include "common.tplvalues.render" ( dict "value" .Values.metrics.jmx.service.annotations "context" $) | nindent 4 }}
+    {{- end }}
+    {{- if .Values.commonAnnotations }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- end }}
+spec:
+  type: {{ .Values.metrics.jmx.service.type }}
+  {{- if eq .Values.metrics.jmx.service.type "LoadBalancer" }}
+  {{- if .Values.metrics.jmx.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.metrics.jmx.service.loadBalancerIP }}
+  {{- end }}
+  {{- if .Values.metrics.jmx.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{- toYaml .Values.metrics.jmx.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  {{- if and (eq .Values.metrics.jmx.service.type "ClusterIP") .Values.metrics.jmx.service.clusterIP }}
+  clusterIP: {{ .Values.metrics.jmx.service.clusterIP }}
+  {{- end }}
+  ports:
+    - name: http-metrics
+      port: {{ .Values.metrics.jmx.service.port }}
+      protocol: TCP
+      targetPort: metrics
+      {{- if and (or (eq .Values.metrics.jmx.service.type "NodePort") (eq .Values.metrics.jmx.service.type "LoadBalancer")) (not (empty .Values.metrics.jmx.service.nodePort)) }}
+      nodePort: {{ .Values.metrics.jmx.service.nodePort }}
+      {{- else if eq .Values.metrics.jmx.service.type "ClusterIP" }}
+      nodePort: null
+      {{- end }}
+  selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/kafka-metrics-deployment.yaml b/kubernetes/helm_charts/kafka/templates/kafka-metrics-deployment.yaml
new file mode 100644
index 0000000000..56657ab473
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/kafka-metrics-deployment.yaml
@@ -0,0 +1,134 @@
+{{- if .Values.metrics.kafka.enabled }}
+{{- $replicaCount := int .Values.replicaCount -}}
+{{- $releaseNamespace := .Release.Namespace -}}
+{{- $clusterDomain := .Values.clusterDomain -}}
+{{- $fullname := include "kafka.fullname" . -}}
+{{- $servicePort := int .Values.service.port -}}
+apiVersion: {{ include "common.capabilities.deployment.apiVersion" . }}
+kind: Deployment
+metadata:
+  name: {{ template "kafka.metrics.kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: metrics
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+      app.kubernetes.io/component: metrics
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app.kubernetes.io/component: metrics
+        {{- if .Values.metrics.kafka.podLabels }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.kafka.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+      annotations:
+        {{- if .Values.metrics.kafka.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.metrics.kafka.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- include "kafka.imagePullSecrets" . | nindent 6 }}
+      {{- if .Values.metrics.kafka.schedulerName }}
+      schedulerName: {{ .Values.metrics.kafka.schedulerName | quote }}
+      {{- end }}
+      serviceAccountName: {{ template "kafka.metrics.kafka.serviceAccountName" . }}
+      {{- if .Values.metrics.kafka.initContainers }}
+      initContainers: {{- include "common.tplvalues.render" (dict "value" .Values.metrics.kafka.initContainers "context" $) | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: kafka-exporter
+          image: {{ include "kafka.metrics.kafka.image" . }}
+          imagePullPolicy: {{ .Values.metrics.kafka.image.pullPolicy | quote }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else }}
+          command:
+            - /bin/bash
+            - -ec
+            - |
+              kafka_exporter \
+              {{- range $i, $e := until $replicaCount }}
+              --kafka.server={{ $fullname }}-{{ $i }}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $servicePort }} \
+              {{- end }}
+              {{- if (include "kafka.client.saslAuthentication" .) }}
+              --sasl.enabled \
+              --sasl.username="$SASL_USERNAME" \
+              --sasl.password="${SASL_USER_PASSWORD%%,*}" \
+              --sasl.mechanism="{{ include "kafka.metrics.kafka.saslMechanism" . }}" \
+              {{- end }}
+              {{- if (include "kafka.client.tlsEncryption" .) }}
+              --tls.enabled \
+              {{- if .Values.metrics.kafka.certificatesSecret }}
+              --tls.key-file="/opt/bitnami/kafka-exporter/certs/{{ .Values.metrics.kafka.tlsKey }}" \
+              --tls.cert-file="/opt/bitnami/kafka-exporter/certs/{{ .Values.metrics.kafka.tlsCert }}" \
+              {{- if .Values.metrics.kafka.tlsCaSecret }}
+              --tls.ca-file="/opt/bitnami/kafka-exporter/cacert/{{ .Values.metrics.kafka.tlsCaCert }}" \
+              {{- else}}
+              --tls.ca-file="/opt/bitnami/kafka-exporter/certs/{{ .Values.metrics.kafka.tlsCaCert }}" \
+              {{- end }}
+              {{- end }}
+              {{- end }}
+              {{- range $key, $value := .Values.metrics.kafka.extraFlags }}
+              --{{ $key }}{{ if $value }}={{ $value }}{{ end }} \
+              {{- end }}
+              --web.listen-address=:9308
+          {{- if (include "kafka.client.saslAuthentication" .) }}
+          {{- $clientUsers := coalesce .Values.auth.sasl.jaas.clientUsers .Values.auth.jaas.clientUsers }}
+          env:
+            - name: SASL_USERNAME
+              value: {{ index $clientUsers 0 | quote }}
+            - name: SASL_USER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "kafka.jaasSecretName" . }}
+                  key: client-passwords
+          {{- end }}
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: 9308
+          {{- if .Values.metrics.kafka.resources }}
+          resources: {{ toYaml .Values.metrics.kafka.resources | nindent 12 }}
+          {{- end }}
+      {{- if and (include "kafka.client.tlsEncryption" .) .Values.metrics.kafka.certificatesSecret }}
+          volumeMounts:
+            - name: kafka-exporter-certificates
+              mountPath: /opt/bitnami/kafka-exporter/certs/
+              readOnly: true
+      {{- if .Values.metrics.kafka.tlsCaSecret }}
+            - name: kafka-exporter-ca-certificate
+              mountPath: /opt/bitnami/kafka-exporter/cacert/
+              readOnly: true
+      {{- end }}
+      volumes:
+        - name: kafka-exporter-certificates
+          secret:
+            secretName: {{ .Values.metrics.kafka.certificatesSecret }}
+            defaultMode: 0440
+      {{- if .Values.metrics.kafka.tlsCaSecret }}
+        - name: kafka-exporter-ca-certificate
+          secret:
+            secretName: {{ .Values.metrics.kafka.tlsCaSecret }}
+            defaultMode: 0440
+      {{- end }}
+      {{- end }}
+      {{- if .Values.metrics.kafka.affinity }}
+      affinity:
+        {{ toYaml .Values.metrics.kafka.affinity | nindent 8 }}
+      {{- end }}
+      {{- if .Values.metrics.kafka.tolerations }}
+      tolerations:
+        {{ toYaml .Values.metrics.kafka.tolerations | nindent 8 }}
+      {{- end }}
+      {{- if .Values.metrics.kafka.nodeSelector }}
+      nodeSelector:
+        {{ toYaml .Values.metrics.kafka.nodeSelector | nindent 8 }}
+      {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/kafka-metrics-serviceaccount.yaml b/kubernetes/helm_charts/kafka/templates/kafka-metrics-serviceaccount.yaml
new file mode 100644
index 0000000000..d68a134628
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/kafka-metrics-serviceaccount.yaml
@@ -0,0 +1,15 @@
+{{- if and .Values.metrics.kafka.enabled .Values.metrics.kafka.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kafka.metrics.kafka.serviceAccountName" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: metrics
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.metrics.kafka.serviceAccount.automountServiceAccountToken }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/kafka-metrics-svc.yaml b/kubernetes/helm_charts/kafka/templates/kafka-metrics-svc.yaml
new file mode 100644
index 0000000000..b123d432f2
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/kafka-metrics-svc.yaml
@@ -0,0 +1,45 @@
+{{- if .Values.metrics.kafka.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kafka.fullname" . }}-metrics
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: metrics
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if or .Values.metrics.kafka.service.annotations .Values.commonAnnotations }}
+  annotations:
+    {{- if .Values.metrics.kafka.service.annotations }}
+    {{ include "common.tplvalues.render" ( dict "value" .Values.metrics.kafka.service.annotations "context" $) | nindent 4 }}
+    {{- end }}
+    {{- if .Values.commonAnnotations }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- end }}
+spec:
+  type: {{ .Values.metrics.kafka.service.type }}
+  {{- if eq .Values.metrics.kafka.service.type "LoadBalancer" }}
+  {{- if .Values.metrics.kafka.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.metrics.kafka.service.loadBalancerIP }}
+  {{- end }}
+  {{- if .Values.metrics.kafka.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{- toYaml .Values.metrics.kafka.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  {{- if and (eq .Values.metrics.kafka.service.type "ClusterIP") .Values.metrics.kafka.service.clusterIP }}
+  clusterIP: {{ .Values.metrics.kafka.service.clusterIP }}
+  {{- end }}
+  ports:
+    - name: http-metrics
+      port: {{ .Values.metrics.kafka.service.port }}
+      protocol: TCP
+      targetPort: metrics
+      {{- if and (or (eq .Values.metrics.kafka.service.type "NodePort") (eq .Values.metrics.kafka.service.type "LoadBalancer")) (not (empty .Values.metrics.kafka.service.nodePort)) }}
+      nodePort: {{ .Values.metrics.kafka.service.nodePort }}
+      {{- else if eq .Values.metrics.kafka.service.type "ClusterIP" }}
+      nodePort: null
+      {{- end }}
+  selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
+    app.kubernetes.io/component: metrics
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/kafka-provisioning.yaml b/kubernetes/helm_charts/kafka/templates/kafka-provisioning.yaml
new file mode 100644
index 0000000000..1af0a3356b
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/kafka-provisioning.yaml
@@ -0,0 +1,161 @@
+{{- if .Values.provisioning.enabled }}
+{{- $replicaCount := int .Values.replicaCount }}
+kind: Job
+apiVersion: batch/v1
+metadata:
+  name: {{ include "kafka.fullname" . }}-provisioning
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka-provisioning
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  annotations:
+    helm.sh/hook: post-install,post-upgrade
+    helm.sh/hook-delete-policy: before-hook-creation
+    {{- if .Values.commonAnnotations }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+spec:
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app.kubernetes.io/component: kafka-provisioning
+        {{- if .Values.podLabels }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+      annotations:
+        {{- if .Values.provisioning.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.provisioning.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
+    spec:
+      {{- include "kafka.imagePullSecrets" . | nindent 6 }}
+      {{- if .Values.provisioning.schedulerName }}
+      schedulerName: {{ .Values.provisioning.schedulerName | quote }}
+      {{- end }}
+      restartPolicy: OnFailure
+      terminationGracePeriodSeconds: 0
+      initContainers:
+        - name: wait-for-available-kafka
+          image: {{ include "kafka.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          command:
+            - /bin/bash
+            - -c
+            - >-
+              set -e;
+              wait-for-port \
+                --host={{ include "kafka.fullname" . }} \
+                --state=inuse \
+                --timeout=120 \
+                {{ .Values.service.port | int64 }};
+              echo "Kafka is available";
+      containers:
+        - name: kafka-provisioning
+          image: {{ include "kafka.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          {{- else }}
+          command:
+            {{- if .Values.provisioning.command }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.provisioning.command "context" $) | nindent 12 }}
+            {{- else }}
+            - /bin/bash
+            {{- end }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else }}
+          args:
+            {{- if .Values.provisioning.args }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.provisioning.args "context" $) | nindent 12 }}
+            {{- else }}
+            - -ec
+            - |
+              {{- $bootstrapServer := printf "%s:%d" (include "kafka.fullname" .) (.Values.service.port | int64) }}
+              {{- range $topic := .Values.provisioning.topics }}
+              echo "Ensure topic '{{ $topic.name }}' exists"
+              /opt/bitnami/kafka/bin/kafka-topics.sh \
+                --create \
+                --if-not-exists \
+                --bootstrap-server {{ $bootstrapServer }} \
+                --replication-factor {{ $topic.replicationFactor | default $.Values.provisioning.replicationFactor }} \
+                --partitions {{ $topic.partitions | default $.Values.provisioning.numPartitions }} \
+                {{- range $name, $value := $topic.config }}
+                --config {{ $name }}={{ $value }} \
+                {{- end }}
+                --topic {{ $topic.name }}
+              {{- end }}
+              echo "Provisioning succeeded"
+            {{- end }}
+          {{- end }}
+          env:
+            - name: BITNAMI_DEBUG
+              value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
+          {{- if .Values.provisioning.resources }}
+          resources: {{- toYaml .Values.provisioning.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            {{- if or .Values.config .Values.existingConfigmap }}
+            - name: kafka-config
+              mountPath: {{ .Values.persistence.mountPath }}/config/server.properties
+              subPath: server.properties
+            {{- end }}
+            {{- if or .Values.log4j .Values.existingLog4jConfigMap }}
+            - name: log4j-config
+              mountPath: {{ .Values.persistence.mountPath }}/config/log4j.properties
+              subPath: log4j.properties
+            {{- end }}
+            {{- if (include "kafka.tlsEncryption" .) }}
+            {{- if not (empty .Values.auth.tls.existingSecrets) }}
+            {{- range $index := .Values.auth.tls.existingSecrets }}
+            - name: kafka-certs-{{ $index }}
+              mountPath: /certs-{{ $index }}
+              readOnly: true
+            {{- end }}
+            {{- else if or (not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret))) (.Files.Glob "files/tls/*.{crt,pem}") }}
+            - name: kafka-certs
+              mountPath: /certs
+              readOnly: true
+            {{- else if .Values.auth.tls.autoGenerated }}
+            {{- range $index := until $replicaCount }}
+            - name: kafka-certs-{{ $index }}
+              mountPath: /certs-{{ $index }}
+              readOnly: true
+            {{- end }}
+            {{- end }}
+            {{- end }}
+      volumes:
+        {{- if or .Values.config .Values.existingConfigmap }}
+        - name: kafka-config
+          configMap:
+            name: {{ include "kafka.configmapName" . }}
+        {{- end }}
+        {{- if or .Values.log4j .Values.existingLog4jConfigMap }}
+        - name: log4j-config
+          configMap:
+            name: {{ include "kafka.log4j.configMapName" . }}
+        {{ end }}
+        {{- if (include "kafka.tlsEncryption" .) }}
+        {{- if not (empty .Values.auth.tls.existingSecrets) }}
+        {{- range $index, $secret := .Values.auth.tls.existingSecrets }}
+        - name: kafka-certs-{{ $index }}
+          secret:
+            secretName: {{ tpl $secret $ }}
+            defaultMode: 256
+        {{- end }}
+        {{- else if or (not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret))) (.Files.Glob "files/tls/*.{crt,pem}") }}
+        - name: kafka-certs
+          secret:
+            secretName: {{ if not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret)) }}{{ tpl (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret) . }}{{ else }}{{ printf "%s-tls" (include "kafka.fullname" .) }}{{ end }}
+            defaultMode: 256
+        {{- else if .Values.auth.tls.autoGenerated }}
+        {{- range $index := until $replicaCount }}
+        - name: kafka-certs-{{ $index }}
+          secret:
+            secretName: {{ printf "%s-%d-tls" (include "kafka.fullname" $) $index }}
+            defaultMode: 256
+        {{- end }}
+        {{- end }}
+        {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/log4j-configmap.yaml b/kubernetes/helm_charts/kafka/templates/log4j-configmap.yaml
new file mode 100644
index 0000000000..90eccde320
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/log4j-configmap.yaml
@@ -0,0 +1,16 @@
+{{- if (include "kafka.log4j.createConfigMap" .) }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "kafka.log4j.configMapName" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  log4j.properties: |-
+    {{- include "common.tplvalues.render" ( dict "value" .Values.log4j "context" $ ) | nindent 4 }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/kafka/templates/poddisruptionbudget.yaml b/kubernetes/helm_charts/kafka/templates/poddisruptionbudget.yaml
new file mode 100644
index 0000000000..30809a0e66
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/poddisruptionbudget.yaml
@@ -0,0 +1,25 @@
+{{- $replicaCount := int .Values.replicaCount }}
+{{- if and .Values.pdb.create (gt $replicaCount 1) }}
+apiVersion: policy/v1beta1
+kind: PodDisruptionBudget
+metadata:
+  name: {{ template "kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if .Values.pdb.minAvailable }}
+  minAvailable: {{ .Values.pdb.minAvailable }}
+  {{- end }}
+  {{- if .Values.pdb.maxUnavailable }}
+  maxUnavailable: {{ .Values.pdb.maxUnavailable }}
+  {{- end }}
+  selector:
+    matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+      app.kubernetes.io/component: kafka
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/role.yaml b/kubernetes/helm_charts/kafka/templates/role.yaml
new file mode 100644
index 0000000000..08e278426e
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/role.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.rbac.create -}}
+apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}
+kind: Role
+metadata:
+  name: {{ template "kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - get
+      - list
+      - watch
+{{- end -}}
diff --git a/kubernetes/helm_charts/kafka/templates/rolebinding.yaml b/kubernetes/helm_charts/kafka/templates/rolebinding.yaml
new file mode 100644
index 0000000000..7c69df0b5b
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/rolebinding.yaml
@@ -0,0 +1,22 @@
+{{- if and .Values.serviceAccount.create .Values.rbac.create }}
+apiVersion: {{ include "common.capabilities.rbac.apiVersion" . }}
+kind: RoleBinding
+metadata:
+  name: {{ template "kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+roleRef:
+  kind: Role
+  name: {{ template "kafka.fullname" . }}
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "kafka.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/scripts-configmap.yaml b/kubernetes/helm_charts/kafka/templates/scripts-configmap.yaml
new file mode 100644
index 0000000000..fad9595594
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/scripts-configmap.yaml
@@ -0,0 +1,160 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "kafka.fullname" . }}-scripts
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+data:
+  {{- $fullname := include "kafka.fullname" . }}
+  {{- $releaseNamespace := .Release.Namespace }}
+  {{- $clusterDomain := .Values.clusterDomain }}
+  {{- $interBrokerPort := .Values.service.internalPort }}
+  {{- $clientPort := .Values.service.port }}
+  {{- $jksTruststoreSecret := coalesce .Values.auth.tls.jksTruststoreSecret .Values.auth.jksTruststoreSecret -}}
+  {{- $jksTruststore := coalesce .Values.auth.tls.jksTruststore .Values.auth.jksTruststore -}}
+  {{- $jksKeystoreSAN := coalesce .Values.auth.tls.jksKeystoreSAN .Values.auth.jksKeystoreSAN -}}
+  {{- if .Values.externalAccess.autoDiscovery.enabled }}
+  auto-discovery.sh: |-
+    #!/bin/bash
+
+    SVC_NAME="${MY_POD_NAME}-external"
+
+    {{- if eq .Values.externalAccess.service.type "LoadBalancer" }}
+    # Auxiliary functions
+    retry_while() {
+        local -r cmd="${1:?cmd is missing}"
+        local -r retries="${2:-12}"
+        local -r sleep_time="${3:-5}"
+        local return_value=1
+
+        read -r -a command <<< "$cmd"
+        for ((i = 1 ; i <= retries ; i+=1 )); do
+            "${command[@]}" && return_value=0 && break
+            sleep "$sleep_time"
+        done
+        return $return_value
+    }
+    k8s_svc_lb_ip() {
+        local namespace=${1:?namespace is missing}
+        local service=${2:?service is missing}
+        local service_ip=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].ip}")
+        local service_hostname=$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.status.loadBalancer.ingress[0].hostname}")
+
+        if [[ -n ${service_ip} ]]; then
+            echo "${service_ip}"
+        else
+            echo "${service_hostname}"
+        fi
+    }
+    k8s_svc_lb_ip_ready() {
+        local namespace=${1:?namespace is missing}
+        local service=${2:?service is missing}
+        [[ -n "$(k8s_svc_lb_ip "$namespace" "$service")" ]]
+    }
+    # Wait until LoadBalancer IP is ready
+    retry_while "k8s_svc_lb_ip_ready {{ $releaseNamespace }} $SVC_NAME" || exit 1
+    # Obtain LoadBalancer external IP
+    k8s_svc_lb_ip "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE"
+    {{- else if eq .Values.externalAccess.service.type "NodePort" }}
+    k8s_svc_node_port() {
+        local namespace=${1:?namespace is missing}
+        local service=${2:?service is missing}
+        local index=${3:-0}
+        local node_port="$(kubectl get svc "$service" -n "$namespace" -o jsonpath="{.spec.ports[$index].nodePort}")"
+        echo "$node_port"
+    }
+    k8s_svc_node_port "{{ $releaseNamespace }}" "$SVC_NAME" | tee "$SHARED_FILE"
+    {{- end }}
+  {{- end }}
+  setup.sh: |-
+    #!/bin/bash
+
+    ID="${MY_POD_NAME#"{{ $fullname }}-"}"
+    if [[ -f "/bitnami/kafka/data/meta.properties" ]]; then
+        export KAFKA_CFG_BROKER_ID="$(grep "broker.id" /bitnami/kafka/data/meta.properties | awk -F '=' '{print $2}')"
+    else
+        export KAFKA_CFG_BROKER_ID="$((ID + {{ .Values.minBrokerId }}))"
+    fi
+
+    {{- if .Values.externalAccess.enabled }}
+    # Configure external ip and port
+    {{- if eq .Values.externalAccess.service.type "LoadBalancer" }}
+    {{- if .Values.externalAccess.autoDiscovery.enabled }}
+    export EXTERNAL_ACCESS_IP="$(<${SHARED_FILE})"
+    {{- else }}
+    export EXTERNAL_ACCESS_IP=$(echo '{{ .Values.externalAccess.service.loadBalancerIPs }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))")
+    {{- end }}
+    export EXTERNAL_ACCESS_PORT={{ .Values.externalAccess.service.port }}
+    {{- else if eq .Values.externalAccess.service.type "NodePort" }}
+    {{- if and .Values.externalAccess.service.usePodIPs .Values.externalAccess.autoDiscovery.enabled }}
+    export EXTERNAL_ACCESS_IP="${MY_POD_IP}"
+    {{- else if or .Values.externalAccess.service.useHostIPs .Values.externalAccess.autoDiscovery.enabled }}
+    export EXTERNAL_ACCESS_IP="${HOST_IP}"
+    {{- else if .Values.externalAccess.service.domain }}
+    export EXTERNAL_ACCESS_IP={{ .Values.externalAccess.service.domain }}
+    {{- else }}
+    export EXTERNAL_ACCESS_IP=$(curl -s https://ipinfo.io/ip)
+    {{- end }}
+    {{- if .Values.externalAccess.autoDiscovery.enabled }}
+    export EXTERNAL_ACCESS_PORT="$(<${SHARED_FILE})"
+    {{- else }}
+    export EXTERNAL_ACCESS_PORT=$(echo '{{ .Values.externalAccess.service.nodePorts }}' | tr -d '[]' | cut -d ' ' -f "$(($ID + 1))")
+    {{- end }}
+    {{- end }}
+
+    # Configure Kafka advertised listeners
+    {{- if .Values.advertisedListeners }}
+    export KAFKA_CFG_ADVERTISED_LISTENERS={{ join "," .Values.advertisedListeners }}
+    {{- else }}
+    export KAFKA_CFG_ADVERTISED_LISTENERS="INTERNAL://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://${MY_POD_NAME}.{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }},EXTERNAL://${EXTERNAL_ACCESS_IP}:${EXTERNAL_ACCESS_PORT}"
+    {{- end }}
+    {{- end }}
+
+    {{- if (include "kafka.tlsEncryption" .) }}
+    mkdir -p /opt/bitnami/kafka/config/certs
+    {{- if eq .Values.auth.tls.type "jks" }}
+    {{- if not (empty .Values.auth.tls.existingSecrets) }}
+    JKS_TRUSTSTORE={{ printf "/%s/%s" (ternary "certs-${ID}" "truststore" (empty $jksTruststoreSecret)) (default "kafka.truststore.jks" $jksTruststore) | quote }}
+    JKS_KEYSTORE="/certs-${ID}/kafka-keystore.jks"
+    {{- else }}
+    JKS_TRUSTSTORE={{ printf "/%s/%s" (ternary "certs" "truststore" (empty $jksTruststoreSecret)) (default "kafka.truststore.jks" $jksTruststore) | quote }}
+    JKS_KEYSTORE={{ printf "/certs/%s" (default "kafka-${ID}.keystore.jks" $jksKeystoreSAN) | quote }}
+    {{- end }}
+    if [[ -f "$JKS_TRUSTSTORE" ]] && [[ -f "$JKS_KEYSTORE" ]]; then
+        cp "$JKS_TRUSTSTORE" "/opt/bitnami/kafka/config/certs/kafka.truststore.jks"
+        cp "$JKS_KEYSTORE" "/opt/bitnami/kafka/config/certs/kafka.keystore.jks"
+    else
+        echo "Couldn't find the expected Java Key Stores (JKS) files! They are mandatory when encryption via TLS is enabled."
+        exit 1
+    fi
+    export KAFKA_TLS_TRUSTSTORE_FILE="/opt/bitnami/kafka/config/certs/kafka.truststore.jks"
+
+    {{- else if eq .Values.auth.tls.type "pem" }}
+    {{- if or (not (empty .Values.auth.tls.existingSecrets)) (and .Values.auth.tls.autoGenerated (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret)) (not (.Files.Glob "files/tls/*.{crt,pem}"))) }}
+    PEM_CA="/certs-${ID}/ca.crt"
+    PEM_CERT="/certs-${ID}/tls.crt"
+    PEM_KEY="/certs-${ID}/tls.key"
+    {{- else }}
+    PEM_CA="/certs/kafka.truststore.pem"
+    PEM_CERT="/certs/kafka-${ID}.keystore.pem"
+    PEM_KEY="/certs/kafka-${ID}.keystore.key"
+    {{- end }}
+    if [[ -f "$PEM_CA" ]] && [[ -f "$PEM_CERT" ]] && [[ -f "$PEM_KEY" ]]; then
+        cp "$PEM_CA" "/opt/bitnami/kafka/config/certs/kafka.truststore.pem"
+        cp "$PEM_CERT" "/opt/bitnami/kafka/config/certs/kafka.keystore.pem"
+        # Ensure the key used PEM format with PKCS#8
+        openssl pkcs8 -topk8 -nocrypt -in "$PEM_KEY" > "/opt/bitnami/kafka/config/certs/kafka.keystore.key"
+    else
+        echo "Couldn't find the expected PEM files! They are mandatory when encryption via TLS is enabled."
+        exit 1
+    fi
+    export KAFKA_TLS_TRUSTSTORE_FILE="/opt/bitnami/kafka/config/certs/kafka.truststore.pem"
+    {{- end }}
+    {{- end }}
+
+    exec /entrypoint.sh /run.sh
diff --git a/kubernetes/helm_charts/kafka/templates/serviceaccount.yaml b/kubernetes/helm_charts/kafka/templates/serviceaccount.yaml
new file mode 100644
index 0000000000..14dc4faeb5
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/serviceaccount.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ template "kafka.serviceAccountName" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+automountServiceAccountToken: {{ .Values.serviceAccount.automountServiceAccountToken }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/servicemonitor-jmx-metrics.yaml b/kubernetes/helm_charts/kafka/templates/servicemonitor-jmx-metrics.yaml
new file mode 100644
index 0000000000..53ede601e5
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/servicemonitor-jmx-metrics.yaml
@@ -0,0 +1,42 @@
+{{- if and .Values.metrics.jmx.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "kafka.fullname" . }}-jmx-metrics
+  {{- if .Values.metrics.serviceMonitor.namespace }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+  {{- end }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+      {{- if .Values.metrics.serviceMonitor.selector }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.metrics.serviceMonitor.selector "context" $) | nindent 6 }}
+      {{- end }}
+      app.kubernetes.io/component: kafka
+  endpoints:
+    - port: http-metrics
+      path: "/"
+      {{- if .Values.metrics.serviceMonitor.interval }}
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.relabelings }}
+      relabelings: {{- toYaml .Values.metrics.serviceMonitor.relabelings | nindent 6 }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.metricRelabelings }}
+      metricRelabelings: {{- toYaml .Values.metrics.serviceMonitor.metricRelabelings | nindent 6 }}
+      {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/servicemonitor-metrics.yaml b/kubernetes/helm_charts/kafka/templates/servicemonitor-metrics.yaml
new file mode 100644
index 0000000000..e5dddb4b81
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/servicemonitor-metrics.yaml
@@ -0,0 +1,42 @@
+{{- if and .Values.metrics.kafka.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "kafka.fullname" . }}-metrics
+  {{- if .Values.metrics.serviceMonitor.namespace }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+  {{- end }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: metrics
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  selector:
+    matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+      {{- if .Values.metrics.serviceMonitor.selector }}
+      {{- include "common.tplvalues.render" (dict "value" .Values.metrics.serviceMonitor.selector "context" $) | nindent 6 }}
+      {{- end }}
+      app.kubernetes.io/component: metrics
+  endpoints:
+    - port: http-metrics
+      path: "/metrics"
+      {{- if .Values.metrics.serviceMonitor.interval }}
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.relabelings }}
+      relabelings: {{- toYaml .Values.metrics.serviceMonitor.relabelings | nindent 6 }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.metricRelabelings }}
+      metricRelabelings: {{- toYaml .Values.metrics.serviceMonitor.metricRelabelings | nindent 6 }}
+      {{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/statefulset.yaml b/kubernetes/helm_charts/kafka/templates/statefulset.yaml
new file mode 100644
index 0000000000..e79fd942ef
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/statefulset.yaml
@@ -0,0 +1,567 @@
+{{- $replicaCount := int .Values.replicaCount }}
+{{- $fullname := include "kafka.fullname" . }}
+{{- $releaseNamespace := .Release.Namespace }}
+{{- $clusterDomain := .Values.clusterDomain }}
+{{- $interBrokerPort := .Values.service.internalPort }}
+{{- $clientPort := .Values.service.port }}
+{{- $interBrokerProtocol := include "kafka.listenerType" (dict "protocol" .Values.auth.interBrokerProtocol) -}}
+{{- $clientProtocol := include "kafka.listenerType" (dict "protocol" .Values.auth.clientProtocol) -}}
+{{- $loadBalancerIPListLength := len .Values.externalAccess.service.loadBalancerIPs }}
+{{- if not (and .Values.externalAccess.enabled (not .Values.externalAccess.autoDiscovery.enabled) (not (eq $replicaCount $loadBalancerIPListLength )) (eq .Values.externalAccess.service.type "LoadBalancer")) }}
+apiVersion: {{ include "common.capabilities.statefulset.apiVersion" . }}
+kind: StatefulSet
+metadata:
+  name: {{ include "kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  podManagementPolicy: {{ .Values.podManagementPolicy }}
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels: {{- include "common.labels.matchLabels" . | nindent 6 }}
+      app.kubernetes.io/component: kafka
+  serviceName: {{ template "kafka.fullname" . }}-headless
+  updateStrategy:
+    type: {{ .Values.updateStrategy | quote }}
+    {{- if (eq "OnDelete" .Values.updateStrategy) }}
+    rollingUpdate: null
+    {{- else if .Values.rollingUpdatePartition }}
+    rollingUpdate:
+      partition: {{ .Values.rollingUpdatePartition }}
+    {{- end }}
+  template:
+    metadata:
+      labels: {{- include "common.labels.standard" . | nindent 8 }}
+        app.kubernetes.io/component: kafka
+        {{- if .Values.podLabels }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.podLabels "context" $) | nindent 8 }}
+        {{- end }}
+      {{- if or (include "kafka.createConfigmap" .) (include "kafka.createJaasSecret" .) .Values.externalAccess.enabled (include "kafka.metrics.jmx.createConfigmap" .) .Values.podAnnotations }}
+      annotations:
+        {{- if (include "kafka.createConfigmap" .) }}
+        checksum/configuration: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+        {{- end }}
+        {{- if (include "kafka.createJaasSecret" .) }}
+        checksum/jaas-secret: {{ include (print $.Template.BasePath "/jaas-secret.yaml") . | sha256sum }}
+        {{- end }}
+         {{- if (include "kafka.createTlsSecret" .) }}
+        checksum/tls-secret: {{ include (print $.Template.BasePath "/tls-secret.yaml") . | sha256sum }}
+        {{- end }}
+        {{- if .Values.externalAccess.enabled }}
+        checksum/scripts: {{ include (print $.Template.BasePath "/scripts-configmap.yaml") . | sha256sum }}
+        {{- end }}
+        {{- if (include "kafka.metrics.jmx.createConfigmap" .) }}
+        checksum/jmx-configuration: {{ include (print $.Template.BasePath "/jmx-configmap.yaml") . | sha256sum }}
+        {{- end }}
+        {{- if .Values.podAnnotations }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }}
+        {{- end }}
+      {{- end }}
+    spec:
+      {{- include "kafka.imagePullSecrets" . | nindent 6 }}
+      {{- if .Values.hostAliases }}
+      hostAliases: {{- include "common.tplvalues.render" (dict "value" .Values.hostAliases "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.schedulerName }}
+      schedulerName: {{ .Values.schedulerName | quote }}
+      {{- end }}
+      {{- if .Values.affinity }}
+      affinity: {{- include "common.tplvalues.render" (dict "value" .Values.affinity "context" $) | nindent 8 }}
+      {{- else }}
+      affinity:
+        podAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAffinityPreset "component" "kafka" "context" $) | nindent 10 }}
+        podAntiAffinity: {{- include "common.affinities.pods" (dict "type" .Values.podAntiAffinityPreset "component" "kafka" "context" $) | nindent 10 }}
+        nodeAffinity: {{- include "common.affinities.nodes" (dict "type" .Values.nodeAffinityPreset.type "key" .Values.nodeAffinityPreset.key "values" .Values.nodeAffinityPreset.values) | nindent 10 }}
+      {{- end }}
+      {{- if .Values.nodeSelector }}
+      nodeSelector: {{- include "common.tplvalues.render" (dict "value" .Values.nodeSelector "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.tolerations }}
+      tolerations: {{- include "common.tplvalues.render" (dict "value" .Values.tolerations "context" $) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.topologySpreadConstraints }}
+      topologySpreadConstraints: {{- include "common.tplvalues.render" (dict "value" .Values.topologySpreadConstraints "context" .) | nindent 8 }}
+      {{- end }}
+      {{- if .Values.terminationGracePeriodSeconds }}
+      terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }}
+      {{- end }}
+      {{- if .Values.priorityClassName }}
+      priorityClassName: {{ .Values.priorityClassName }}
+      {{- end }}
+      {{- if .Values.podSecurityContext.enabled }}
+      securityContext: {{- omit .Values.podSecurityContext "enabled" | toYaml | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ template "kafka.serviceAccountName" . }}
+      {{- if or (and .Values.volumePermissions.enabled .Values.persistence.enabled) (and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled) .Values.initContainers }}
+      initContainers:
+        {{- if and .Values.volumePermissions.enabled .Values.persistence.enabled }}
+        - name: volume-permissions
+          image: {{ include "kafka.volumePermissions.image" . }}
+          imagePullPolicy: {{ .Values.volumePermissions.image.pullPolicy | quote }}
+          command:
+            - /bin/bash
+          args:
+            - -ec
+            - |
+              chown -R "{{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }}" "{{ .Values.persistence.mountPath }}"
+              chown -R "{{ .Values.podSecurityContext.runAsUser }}:{{ .Values.podSecurityContext.fsGroup }}" "{{ .Values.logPersistence.mountPath }}"
+          securityContext: {{- .Values.volumePermissions.securityContext | toYaml | nindent 12 }}
+          {{- if .Values.volumePermissions.resources }}
+          resources: {{- toYaml .Values.volumePermissions.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: data
+              mountPath: {{ .Values.persistence.mountPath }}
+            - name: logs
+              mountPath: {{ .Values.logPersistence.mountPath }}
+        {{- end }}
+        {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }}
+        - name: auto-discovery
+          image: {{ include "kafka.externalAccess.autoDiscovery.image" . }}
+          imagePullPolicy: {{ .Values.externalAccess.autoDiscovery.image.pullPolicy | quote }}
+          command:
+            - /scripts/auto-discovery.sh
+          env:
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: SHARED_FILE
+              value: "/shared/info.txt"
+          {{- if .Values.externalAccess.autoDiscovery.resources }}
+          resources: {{- toYaml .Values.externalAccess.autoDiscovery.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: shared
+              mountPath: /shared
+            - name: logs
+              mountPath: {{ .Values.logPersistence.mountPath }}
+            - name: scripts
+              mountPath: /scripts/auto-discovery.sh
+              subPath: auto-discovery.sh
+        {{- end }}
+        {{- if .Values.initContainers }}
+        {{- include "common.tplvalues.render" ( dict "value" .Values.initContainers "context" $ ) | nindent 8 }}
+        {{- end }}
+      {{- end }}
+      containers:
+        - name: kafka
+          image: {{ include "kafka.image" . }}
+          imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          {{- else if .Values.command }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.command "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.diagnosticMode.enabled }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else if .Values.args }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.args "context" $) | nindent 12 }}
+          {{- end }}
+          env:
+            - name: BITNAMI_DEBUG
+              value: {{ ternary "true" "false" (or .Values.image.debug .Values.diagnosticMode.enabled) | quote }}
+            - name: MY_POD_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.podIP
+            - name: MY_POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: KAFKA_CFG_ZOOKEEPER_CONNECT
+              {{- if .Values.zookeeper.enabled }}
+              value: {{ include "kafka.zookeeper.fullname" . | quote }}
+              {{- else }}
+              value: {{ include "common.tplvalues.render" (dict "value" (join "," .Values.externalZookeeper.servers) "context" $) }}
+              {{- end }}
+            - name: KAFKA_INTER_BROKER_LISTENER_NAME
+              value: {{ .Values.interBrokerListenerName | quote }}
+            - name: KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP
+              {{- if .Values.listenerSecurityProtocolMap }}
+              value: {{ .Values.listenerSecurityProtocolMap | quote }}
+              {{- else if .Values.externalAccess.enabled }}
+              value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }},EXTERNAL:{{ $clientProtocol }}"
+              {{- else }}
+              value: "INTERNAL:{{ $interBrokerProtocol }},CLIENT:{{ $clientProtocol }}"
+              {{- end }}
+            {{- if or ($clientProtocol | regexFind "SASL") ($interBrokerProtocol | regexFind "SASL") (coalesce .Values.auth.sasl.jaas.zookeeperUser .Values.auth.jaas.zookeeperUser) }}
+            - name: KAFKA_CFG_SASL_ENABLED_MECHANISMS
+              value: {{ upper (coalesce .Values.auth.sasl.mechanisms .Values.auth.saslMechanisms) | quote }}
+            - name: KAFKA_CFG_SASL_MECHANISM_INTER_BROKER_PROTOCOL
+              value: {{ upper (coalesce .Values.auth.sasl.interBrokerMechanism .Values.auth.saslInterBrokerMechanism) | quote }}
+            {{- end }}
+            - name: KAFKA_CFG_LISTENERS
+              {{- if .Values.listeners }}
+              value: {{ join "," .Values.listeners }}
+              {{- else if .Values.externalAccess.enabled }}
+              value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092,EXTERNAL://:9094"
+              {{- else }}
+              value: "INTERNAL://:{{ $interBrokerPort }},CLIENT://:9092"
+              {{- end }}
+            {{- if .Values.externalAccess.enabled }}
+            {{- if .Values.externalAccess.autoDiscovery.enabled }}
+            - name: SHARED_FILE
+              value: "/shared/info.txt"
+            {{- end }}
+            {{- if eq .Values.externalAccess.service.type "NodePort" }}
+            - name: HOST_IP
+              valueFrom:
+                fieldRef:
+                  fieldPath: status.hostIP
+            {{- end }}
+            {{- else }}
+            - name: KAFKA_CFG_ADVERTISED_LISTENERS
+              {{- if .Values.advertisedListeners }}
+              value: {{ join "," .Values.advertisedListeners }}
+              {{- else }}
+              value: "INTERNAL://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $interBrokerPort }},CLIENT://$(MY_POD_NAME).{{ $fullname }}-headless.{{ $releaseNamespace }}.svc.{{ $clusterDomain }}:{{ $clientPort }}"
+              {{- end }}
+            {{- end }}
+            - name: ALLOW_PLAINTEXT_LISTENER
+              value: {{ ternary "yes" "no" .Values.allowPlaintextListener | quote }}
+            {{- if or (include "kafka.client.saslAuthentication" .) (include "kafka.interBroker.saslAuthentication" .) }}
+            - name: KAFKA_OPTS
+              value: "-Djava.security.auth.login.config=/opt/bitnami/kafka/config/kafka_jaas.conf"
+            {{- if (include "kafka.client.saslAuthentication" .) }}
+            - name: KAFKA_CLIENT_USERS
+              value: {{ join "," (coalesce .Values.auth.sasl.jaas.clientUsers .Values.auth.jaas.clientUsers) | quote }}
+            - name: KAFKA_CLIENT_PASSWORDS
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "kafka.jaasSecretName" . }}
+                  key: client-passwords
+            {{- end }}
+            {{- if (include "kafka.interBroker.saslAuthentication" .) }}
+            - name: KAFKA_INTER_BROKER_USER
+              value: {{ coalesce .Values.auth.sasl.jaas.interBrokerUser .Values.auth.jaas.interBrokerUser | quote }}
+            - name: KAFKA_INTER_BROKER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "kafka.jaasSecretName" . }}
+                  key: inter-broker-password
+            {{- end }}
+            {{- end }}
+            {{- if and .Values.zookeeper.auth.enabled (coalesce .Values.auth.sasl.jaas.zookeeperUser .Values.auth.jaas.zookeeperUser) }}
+            - name: KAFKA_ZOOKEEPER_PROTOCOL
+              value: "SASL"
+            - name: KAFKA_ZOOKEEPER_USER
+              value: {{ coalesce .Values.auth.sasl.jaas.zookeeperUser .Values.auth.jaas.zookeeperUser | quote }}
+            - name: KAFKA_ZOOKEEPER_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: {{ include "kafka.jaasSecretName" . }}
+                  key: zookeeper-password
+            {{- end }}
+            {{- if (include "kafka.tlsEncryption" .) }}
+            - name: KAFKA_TLS_TYPE
+              value: {{ upper .Values.auth.tls.type | quote }}
+            - name: KAFKA_CFG_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM
+              value: {{ default "" (coalesce .Values.auth.tls.endpointIdentificationAlgorithm .Values.auth.tlsEndpointIdentificationAlgorithm) | quote }}
+            - name: KAFKA_TLS_CLIENT_AUTH
+              value: {{ ternary "required" "none" (eq .Values.auth.clientProtocol "mtls") | quote }}
+            {{- $tlsPassword := coalesce .Values.auth.tls.password .Values.auth.jksPassword }}
+            - name: KAFKA_CERTIFICATE_PASSWORD
+              value: {{ default "" $tlsPassword | quote }}
+            {{- end }}
+            {{- if .Values.metrics.jmx.enabled }}
+            - name: JMX_PORT
+              value: "5555"
+            {{- end }}
+            - name: KAFKA_VOLUME_DIR
+              value: {{ .Values.persistence.mountPath | quote }}
+            - name: KAFKA_LOG_DIR
+              value: {{ .Values.logPersistence.mountPath | quote }}
+            - name: KAFKA_CFG_DELETE_TOPIC_ENABLE
+              value: {{ .Values.deleteTopicEnable | quote }}
+            - name: KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE
+              value: {{ .Values.autoCreateTopicsEnable | quote }}
+            - name: KAFKA_HEAP_OPTS
+              value: {{ .Values.heapOpts | quote }}
+            - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MESSAGES
+              value: {{ .Values.logFlushIntervalMessages | replace "_" "" | quote }}
+            - name: KAFKA_CFG_LOG_FLUSH_INTERVAL_MS
+              value: {{ .Values.logFlushIntervalMs | quote }}
+            - name: KAFKA_CFG_LOG_RETENTION_BYTES
+              value: {{ .Values.logRetentionBytes | replace "_" "" | quote }}
+            - name: KAFKA_CFG_LOG_RETENTION_CHECK_INTERVALS_MS
+              value: {{ .Values.logRetentionCheckIntervalMs | quote }}
+            - name: KAFKA_CFG_LOG_RETENTION_HOURS
+              value: {{ .Values.logRetentionHours | quote }}
+            - name: KAFKA_CFG_MESSAGE_MAX_BYTES
+              value: {{ .Values.maxMessageBytes | replace "_" "" | quote }}
+            - name: KAFKA_CFG_LOG_SEGMENT_BYTES
+              value: {{ .Values.logSegmentBytes | replace "_" "" | quote }}
+            - name: KAFKA_CFG_LOG_DIRS
+              value: {{ .Values.logsDirs | quote }}
+            - name: KAFKA_CFG_DEFAULT_REPLICATION_FACTOR
+              value: {{ .Values.defaultReplicationFactor | quote }}
+            - name: KAFKA_CFG_OFFSETS_TOPIC_REPLICATION_FACTOR
+              value: {{ .Values.offsetsTopicReplicationFactor | quote }}
+            - name: KAFKA_CFG_TRANSACTION_STATE_LOG_REPLICATION_FACTOR
+              value: {{ .Values.transactionStateLogReplicationFactor | quote }}
+            - name: KAFKA_CFG_TRANSACTION_STATE_LOG_MIN_ISR
+              value: {{ .Values.transactionStateLogMinIsr | quote }}
+            - name: KAFKA_CFG_NUM_IO_THREADS
+              value: {{ .Values.numIoThreads | quote }}
+            - name: KAFKA_CFG_NUM_NETWORK_THREADS
+              value: {{ .Values.numNetworkThreads | quote }}
+            - name: KAFKA_CFG_NUM_PARTITIONS
+              value: {{ .Values.numPartitions | quote }}
+            - name: KAFKA_CFG_NUM_RECOVERY_THREADS_PER_DATA_DIR
+              value: {{ .Values.numRecoveryThreadsPerDataDir | quote }}
+            - name: KAFKA_CFG_SOCKET_RECEIVE_BUFFER_BYTES
+              value: {{ .Values.socketReceiveBufferBytes | quote }}
+            - name: KAFKA_CFG_SOCKET_REQUEST_MAX_BYTES
+              value: {{ .Values.socketRequestMaxBytes | replace "_" "" | quote }}
+            - name: KAFKA_CFG_SOCKET_SEND_BUFFER_BYTES
+              value: {{ .Values.socketSendBufferBytes | quote }}
+            - name: KAFKA_CFG_ZOOKEEPER_CONNECTION_TIMEOUT_MS
+              value: {{ .Values.zookeeperConnectionTimeoutMs | quote }}
+            - name: KAFKA_CFG_AUTHORIZER_CLASS_NAME
+              value: {{ .Values.authorizerClassName | quote }}
+            - name: KAFKA_CFG_ALLOW_EVERYONE_IF_NO_ACL_FOUND
+              value: {{ .Values.allowEveryoneIfNoAclFound | quote }}
+            - name: KAFKA_CFG_SUPER_USERS
+              value: {{ .Values.superUsers | quote }}
+            {{- if .Values.extraEnvVars }}
+            {{ include "common.tplvalues.render" ( dict "value" .Values.extraEnvVars "context" $) | nindent 12 }}
+            {{- end }}
+          ports:
+            - name: kafka-client
+              containerPort: 9092
+            - name: kafka-internal
+              containerPort: {{ $interBrokerPort }}
+            {{- if .Values.externalAccess.enabled }}
+            - name: kafka-external
+              containerPort: 9094
+            {{- end }}
+          {{- if not .Values.diagnosticMode.enabled }}
+          {{- if .Values.livenessProbe.enabled }}
+          livenessProbe:
+            tcpSocket:
+              port: kafka-client
+            initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }}
+            timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.livenessProbe.failureThreshold }}
+            periodSeconds: {{ .Values.livenessProbe.periodSeconds }}
+            successThreshold: {{ .Values.livenessProbe.successThreshold }}
+          {{- else if .Values.customLivenessProbe }}
+          livenessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customLivenessProbe "context" $) | nindent 12 }}
+          {{- end }}
+          {{- if .Values.readinessProbe.enabled }}
+          readinessProbe:
+            tcpSocket:
+              port: kafka-client
+            initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }}
+            timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }}
+            failureThreshold: {{ .Values.readinessProbe.failureThreshold }}
+            periodSeconds: {{ .Values.readinessProbe.periodSeconds }}
+            successThreshold: {{ .Values.readinessProbe.successThreshold }}
+          {{- else if .Values.customReadinessProbe }}
+          readinessProbe: {{- include "common.tplvalues.render" (dict "value" .Values.customReadinessProbe "context" $) | nindent 12 }}
+          {{- end }}
+          {{- end }}
+          {{- if .Values.resources }}
+          resources: {{- toYaml .Values.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: data
+              mountPath: {{ .Values.persistence.mountPath }}
+            - name: logs
+              mountPath: {{ .Values.logPersistence.mountPath }}
+            {{- if or .Values.config .Values.existingConfigmap }}
+            - name: kafka-config
+              mountPath: {{ .Values.persistence.mountPath }}/config/server.properties
+              subPath: server.properties
+            {{- end }}
+            {{- if or .Values.log4j .Values.existingLog4jConfigMap }}
+            - name: log4j-config
+              mountPath: {{ .Values.persistence.mountPath }}/config/log4j.properties
+              subPath: log4j.properties
+            {{- end }}
+            - name: scripts
+              mountPath: /scripts/setup.sh
+              subPath: setup.sh
+            {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }}
+            - name: shared
+              mountPath: /shared
+            {{- end }}
+            {{- if (include "kafka.tlsEncryption" .) }}
+            {{- if not (empty .Values.auth.tls.existingSecrets) }}
+            {{- range $index, $_ := .Values.auth.tls.existingSecrets }}
+            - name: kafka-certs-{{ $index }}
+              mountPath: /certs-{{ $index }}
+              readOnly: true
+            {{- end }}
+            {{- else if or (not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret))) (.Files.Glob "files/tls/*.{crt,pem}") }}
+            - name: kafka-certs
+              mountPath: /certs
+              readOnly: true
+            {{- else if .Values.auth.tls.autoGenerated }}
+            {{- range $index := until $replicaCount }}
+            - name: kafka-certs-{{ $index }}
+              mountPath: /certs-{{ $index }}
+              readOnly: true
+            {{- end }}
+            {{- end }}
+            {{- if (coalesce .Values.auth.tls.jksTruststoreSecret .Values.auth.jksTruststoreSecret) }}
+            - name: kafka-truststore
+              mountPath: /truststore
+              readOnly: true
+            {{- end }}
+            {{- end }}
+            {{- if .Values.extraVolumeMounts }}
+            {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumeMounts "context" $) | nindent 12 }}
+            {{- end }}
+        {{- if .Values.metrics.jmx.enabled }}
+        - name: jmx-exporter
+          image: {{ template "kafka.metrics.jmx.image" . }}
+          imagePullPolicy: {{ .Values.metrics.jmx.image.pullPolicy | quote }}
+          {{- if .Values.diagnosticMode.enabled }}
+          command: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.command "context" $) | nindent 12 }}
+          args: {{- include "common.tplvalues.render" (dict "value" .Values.diagnosticMode.args "context" $) | nindent 12 }}
+          {{- else }}
+          command:
+            - java
+            - -XX:+UnlockExperimentalVMOptions
+            - -XX:+UseCGroupMemoryLimitForHeap
+            - -XX:MaxRAMFraction=1
+            - -XshowSettings:vm
+            - -jar
+            - jmx_prometheus_httpserver.jar
+            - "5556"
+            - /etc/jmx-kafka/jmx-kafka-prometheus.yml
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: 5556
+          {{- if .Values.metrics.jmx.resources }}
+          resources: {{- toYaml .Values.metrics.jmx.resources | nindent 12 }}
+          {{- end }}
+          volumeMounts:
+            - name: jmx-config
+              mountPath: /etc/jmx-kafka
+        {{- end }}
+        {{- if .Values.sidecars }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.sidecars "context" $) | nindent 8 }}
+        {{- end }}
+      volumes:
+        {{- if or .Values.config .Values.existingConfigmap }}
+        - name: kafka-config
+          configMap:
+            name: {{ include "kafka.configmapName" . }}
+        {{- end }}
+        {{- if or .Values.log4j .Values.existingLog4jConfigMap }}
+        - name: log4j-config
+          configMap:
+            name: {{ include "kafka.log4j.configMapName" . }}
+        {{ end }}
+        - name: scripts
+          configMap:
+            name: {{ include "kafka.fullname" . }}-scripts
+            defaultMode: 0755
+        {{- if and .Values.externalAccess.enabled .Values.externalAccess.autoDiscovery.enabled }}
+        - name: shared
+          emptyDir: {}
+        {{- end }}
+        {{- if .Values.metrics.jmx.enabled }}
+        - name: jmx-config
+          configMap:
+            name: {{ include "kafka.metrics.jmx.configmapName" . }}
+        {{- end }}
+        {{- if (include "kafka.tlsEncryption" .) }}
+        {{- if not (empty .Values.auth.tls.existingSecrets) }}
+        {{- range $index, $secret := .Values.auth.tls.existingSecrets }}
+        - name: kafka-certs-{{ $index }}
+          secret:
+            secretName: {{ tpl $secret $ }}
+            defaultMode: 256
+        {{- end }}
+        {{- else if or (not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret))) (.Files.Glob "files/tls/*.{crt,pem}") }}
+        - name: kafka-certs
+          secret:
+            secretName: {{ if not (empty (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret)) }}{{ tpl (coalesce .Values.auth.tls.existingSecret .Values.auth.jksSecret) . }}{{ else }}{{ printf "%s-tls" (include "kafka.fullname" .) }}{{ end }}
+            defaultMode: 256
+        {{- else if .Values.auth.tls.autoGenerated }}
+        {{- range $index := until $replicaCount }}
+        - name: kafka-certs-{{ $index }}
+          secret:
+            secretName: {{ printf "%s-%d-tls" (include "kafka.fullname" $) $index }}
+            defaultMode: 256
+        {{- end }}
+        {{- end }}
+        {{- if (coalesce .Values.auth.tls.jksTruststoreSecret .Values.auth.jksTruststoreSecret) }}
+        - name: kafka-truststore
+          secret:
+            secretName: {{ coalesce .Values.auth.tls.jksTruststoreSecret .Values.auth.jksTruststoreSecret }}
+            defaultMode: 256
+        {{- end }}
+        {{- end }}
+        {{- if .Values.extraVolumes }}
+        {{- include "common.tplvalues.render" (dict "value" .Values.extraVolumes "context" $) | nindent 8 }}
+        {{- end }}
+{{- if not .Values.persistence.enabled }}
+        - name: data
+          emptyDir: {}
+{{- else if .Values.persistence.existingClaim }}
+        - name: data
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.persistence.existingClaim .) }}
+{{- end }}
+{{- if not .Values.logPersistence.enabled }}
+        - name: logs
+          emptyDir: {}
+{{- else if .Values.logPersistence.existingClaim }}
+        - name: logs
+          persistentVolumeClaim:
+            claimName: {{ printf "%s" (tpl .Values.logPersistence.existingClaim .) }}
+{{- end }}
+  {{- if or (and .Values.persistence.enabled (not .Values.persistence.existingClaim)) (and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim)) }}
+  volumeClaimTemplates:
+  {{- end }}
+{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }}
+    - metadata:
+        name: data
+        {{- if .Values.persistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+        {{- range .Values.persistence.accessModes }}
+          - {{ . | quote }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.persistence.size | quote }}
+        {{ include "kafka.storageClass" . | nindent 8 }}
+        {{- if .Values.persistence.selector }}
+        selector: {{- include "common.tplvalues.render" (dict "value" .Values.persistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+{{- end }}
+{{- if and .Values.logPersistence.enabled (not .Values.logPersistence.existingClaim) }}
+    - metadata:
+        name: logs
+        {{- if .Values.logPersistence.annotations }}
+        annotations: {{- include "common.tplvalues.render" (dict "value" .Values.logPersistence.annotations "context" $) | nindent 10 }}
+        {{- end }}
+      spec:
+        accessModes:
+        {{- range .Values.logPersistence.accessModes }}
+          - {{ . | quote }}
+        {{- end }}
+        resources:
+          requests:
+            storage: {{ .Values.logPersistence.size | quote }}
+        {{ include "kafka.storageClass" . | nindent 8 }}
+        {{- if .Values.logPersistence.selector }}
+        selector: {{- include "common.tplvalues.render" (dict "value" .Values.logPersistence.selector "context" $) | nindent 10 }}
+        {{- end -}}
+{{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/svc-external-access.yaml b/kubernetes/helm_charts/kafka/templates/svc-external-access.yaml
new file mode 100644
index 0000000000..9169a655cb
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/svc-external-access.yaml
@@ -0,0 +1,53 @@
+{{- if .Values.externalAccess.enabled }}
+{{- $fullName := include "kafka.fullname" . }}
+{{- $replicaCount := .Values.replicaCount | int }}
+{{- $root := . }}
+
+{{- range $i, $e := until $replicaCount }}
+{{- $targetPod := printf "%s-%d" (printf "%s" $fullName) $i }}
+{{- $_ := set $ "targetPod" $targetPod }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kafka.fullname" $ }}-{{ $i }}-external
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    pod: {{ $targetPod }}
+    {{- if $root.Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" $root.Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if or $root.Values.externalAccess.service.annotations $root.Values.commonAnnotations }}
+  annotations:
+    {{- if $root.Values.externalAccess.service.annotations }}
+    {{ include "common.tplvalues.render" ( dict "value" $root.Values.externalAccess.service.annotations "context" $) | nindent 4 }}
+    {{- end }}
+    {{- if $root.Values.commonAnnotations }}
+    {{- include "common.tplvalues.render" ( dict "value" $root.Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- end }}
+spec:
+  type: {{ $root.Values.externalAccess.service.type }}
+  {{- if eq $root.Values.externalAccess.service.type "LoadBalancer" }}
+  {{- if not (empty $root.Values.externalAccess.service.loadBalancerIPs) }}
+  loadBalancerIP: {{ index $root.Values.externalAccess.service.loadBalancerIPs $i }}
+  {{- end }}
+  {{- if $root.Values.externalAccess.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{- toYaml $root.Values.externalAccess.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  ports:
+    - name: tcp-kafka
+      port: {{ $root.Values.externalAccess.service.port }}
+      {{- if not (empty $root.Values.externalAccess.service.nodePorts) }}
+      nodePort: {{ index $root.Values.externalAccess.service.nodePorts $i }}
+      {{- else }}
+      nodePort: null
+      {{- end }}
+      targetPort: kafka-external
+  selector: {{- include "common.labels.matchLabels" $ | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    statefulset.kubernetes.io/pod-name: {{ $targetPod }}
+---
+{{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/templates/svc-headless.yaml b/kubernetes/helm_charts/kafka/templates/svc-headless.yaml
new file mode 100644
index 0000000000..ad2303a946
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/svc-headless.yaml
@@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kafka.fullname" . }}-headless
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+spec:
+  type: ClusterIP
+  clusterIP: None
+  ports:
+    - name: tcp-client
+      port: {{ .Values.service.port }}
+      protocol: TCP
+      targetPort: kafka-client
+    - name: tcp-internal
+      port: {{ .Values.service.internalPort }}
+      protocol: TCP
+      targetPort: kafka-internal
+  selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
diff --git a/kubernetes/helm_charts/kafka/templates/svc.yaml b/kubernetes/helm_charts/kafka/templates/svc.yaml
new file mode 100644
index 0000000000..70b24dd9d6
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/svc.yaml
@@ -0,0 +1,49 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ template "kafka.fullname" . }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if or .Values.service.annotations .Values.commonAnnotations }}
+  annotations:
+    {{- if .Values.service.annotations }}
+    {{ include "common.tplvalues.render" ( dict "value" .Values.service.annotations "context" $) | nindent 4 }}
+    {{- end }}
+    {{- if .Values.commonAnnotations }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- end }}
+spec:
+  type: {{ .Values.service.type }}
+  {{- if eq .Values.service.type "LoadBalancer" }}
+  {{- if .Values.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.service.loadBalancerIP }}
+  {{- end }}
+  {{- if .Values.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges: {{- toYaml .Values.service.loadBalancerSourceRanges | nindent 4 }}
+  {{- end }}
+  {{- end }}
+  ports:
+    - name: tcp-client
+      port: {{ .Values.service.port }}
+      protocol: TCP
+      targetPort: kafka-client
+      {{- if and (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) (not (empty .Values.service.nodePorts.client)) }}
+      nodePort: {{ .Values.service.nodePorts.client }}
+      {{- else if eq .Values.service.type "ClusterIP" }}
+      nodePort: null
+      {{- end }}
+    {{- if and .Values.externalAccess.enabled (or (eq .Values.service.type "NodePort") (eq .Values.service.type "LoadBalancer")) }}
+    - name: tcp-external
+      port: {{ .Values.service.externalPort }}
+      protocol: TCP
+      targetPort: kafka-external
+      {{- if (not (empty .Values.service.nodePorts.external)) }}
+      nodePort: {{ .Values.service.nodePorts.external }}
+      {{- end }}
+    {{- end }}
+  selector: {{- include "common.labels.matchLabels" . | nindent 4 }}
+    app.kubernetes.io/component: kafka
diff --git a/kubernetes/helm_charts/kafka/templates/tls-secret.yaml b/kubernetes/helm_charts/kafka/templates/tls-secret.yaml
new file mode 100644
index 0000000000..894adc85a8
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/templates/tls-secret.yaml
@@ -0,0 +1,49 @@
+{{- if (include "kafka.createTlsSecret" .) }}
+{{- if .Files.Glob "files/tls/*.{crt,pem,jks}" }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-tls" (include "kafka.fullname" .) }}
+  labels: {{- include "common.labels.standard" . | nindent 4 }}
+    {{- if .Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" .Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if .Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+type: Opaque
+data:
+  {{- $root := . }}
+  {{- range $path, $bytes := .Files.Glob "files/tls/*.{crt,pem,jks}" }}
+  {{ base $path }}: {{ $root.Files.Get $path | b64enc | quote }}
+  {{- end }}
+{{- else if and .Values.auth.tls.autoGenerated (eq .Values.auth.tls.type "pem") }}
+{{- $replicaCount := int .Values.replicaCount }}
+{{- $releaseNamespace := .Release.Namespace }}
+{{- $clusterDomain := .Values.clusterDomain }}
+{{- $fullname := include "kafka.fullname" . }}
+{{- $ca := genCA "kafka-ca" 365 }}
+{{- range $i := until $replicaCount }}
+{{- $replicaHost := printf "%s-%d.%s-headless" $fullname $i $fullname }}
+{{- $altNames := list (printf "%s.%s.svc.%s" $replicaHost $releaseNamespace $clusterDomain) (printf "%s.%s.svc.%s" $fullname $releaseNamespace $clusterDomain) (printf "%s.%s" $replicaHost $releaseNamespace) (printf "%s.%s" $fullname $releaseNamespace) $replicaHost $fullname }}
+{{- $cert := genSignedCert $replicaHost nil $altNames 365 $ca }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ printf "%s-%d-tls" (include "kafka.fullname" $) $i }}
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+    {{- if $.Values.commonLabels }}
+    {{- include "common.tplvalues.render" ( dict "value" $.Values.commonLabels "context" $ ) | nindent 4 }}
+    {{- end }}
+  {{- if $.Values.commonAnnotations }}
+  annotations: {{- include "common.tplvalues.render" ( dict "value" $.Values.commonAnnotations "context" $ ) | nindent 4 }}
+  {{- end }}
+type: kubernetes.io/tls
+data:
+  ca.crt: {{ $ca.Cert | b64enc | quote }}
+  tls.crt: {{ $cert.Cert | b64enc | quote }}
+  tls.key: {{ $cert.Key | b64enc | quote }}
+---
+{{- end }}
+{{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/kafka/values.j2 b/kubernetes/helm_charts/kafka/values.j2
new file mode 100644
index 0000000000..f303f82daa
--- /dev/null
+++ b/kubernetes/helm_charts/kafka/values.j2
@@ -0,0 +1,1380 @@
+## @section Global parameters
+## Global Docker image parameters
+## Please, note that this will override the image parameters, including dependencies, configured to use the global value
+## Current available global Docker image parameters: imageRegistry, imagePullSecrets and storageClass
+
+## @param global.imageRegistry Global Docker image registry
+## @param global.imagePullSecrets Global Docker registry secret names as an array
+## @param global.storageClass Global StorageClass for Persistent Volume(s)
+##
+global:
+  imageRegistry: ""
+  ## E.g.
+  ## imagePullSecrets:
+  ##   - myRegistryKeySecretName
+  ##
+  imagePullSecrets: []
+  storageClass: ""
+
+## @section Common parameters
+
+## @param nameOverride String to partially override kafka.fullname
+##
+nameOverride: ""
+## @param fullnameOverride String to fully override kafka.fullname
+##
+fullnameOverride: ""
+## @param clusterDomain Default Kubernetes cluster domain
+##
+clusterDomain: cluster.local
+## @param commonLabels Labels to add to all deployed objects
+##
+commonLabels: {}
+## @param commonAnnotations Annotations to add to all deployed objects
+##
+commonAnnotations: {}
+## @param extraDeploy Array of extra objects to deploy with the release
+##
+extraDeploy: []
+
+## Enable diagnostic mode in the deployment
+##
+diagnosticMode:
+  ## @param diagnosticMode.enabled Enable diagnostic mode (all probes will be disabled and the command will be overridden)
+  ##
+  enabled: false
+  ## @param diagnosticMode.command Command to override all containers in the deployment
+  ##
+  command:
+    - sleep
+  ## @param diagnosticMode.args Args to override all containers in the deployment
+  ##
+  args:
+    - infinity
+
+## @section Kafka parameters
+
+## Bitnami Kafka image version
+## ref: https://hub.docker.com/r/bitnami/kafka/tags/
+## @param image.registry Kafka image registry
+## @param image.repository Kafka image repository
+## @param image.tag Kafka image tag (immutable tags are recommended)
+## @param image.pullPolicy Kafka image pull policy
+## @param image.pullSecrets Specify docker-registry secret names as an array
+## @param image.debug Set to true if you would like to see extra information on logs
+##
+image:
+  registry: docker.io
+  repository: "{{ kafka_image_repository }}"
+  tag: "{{ kafka_image_tag }}"
+  ## Specify a imagePullPolicy
+  ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+  ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+  ##
+  pullPolicy: Always
+  ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace)
+  ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+  ## Example:
+  ## pullSecrets:
+  ##   - myRegistryKeySecretName
+  ##
+  pullSecrets: []
+  ## Set to true if you would like to see extra information on logs
+  ##
+  debug: false
+## @param config Configuration file for Kafka. Auto-generated based on other parameters when not specified (see [below](
+## Specify content for server.properties
+## NOTE: This will override any KAFKA_CFG_ environment variables (including those set by the chart)
+## The server.properties is auto-generated based on other parameters when this parameter is not specified
+##
+## Example:
+## config: |-
+##   broker.id=-1
+##   listeners=PLAINTEXT://:9092
+##   advertised.listeners=PLAINTEXT://KAFKA_IP:9092
+##   num.network.threads=3
+##   num.io.threads=8
+##   socket.send.buffer.bytes=102400
+##   socket.receive.buffer.bytes=102400
+##   socket.request.max.bytes=104857600
+##   log.dirs=/bitnami/kafka/data
+##   num.partitions=1
+##   num.recovery.threads.per.data.dir=1
+##   offsets.topic.replication.factor=1
+##   transaction.state.log.replication.factor=1
+##   transaction.state.log.min.isr=1
+##   log.flush.interval.messages=10000
+##   log.flush.interval.ms=1000
+##   log.retention.hours=168
+##   log.retention.bytes=1073741824
+##   log.segment.bytes=1073741824
+##   log.retention.check.interval.ms=300000
+##   zookeeper.connect=ZOOKEEPER_SERVICE_NAME
+##   zookeeper.connection.timeout.ms=6000
+##   group.initial.rebalance.delay.ms=0
+##
+config: ""
+## @param existingConfigmap ConfigMap with Kafka Configuration
+## NOTE: This will override config AND any KAFKA_CFG_ environment variables.
+##
+existingConfigmap: ""
+## @param log4j An optional log4j.properties file to overwrite the default of the Kafka brokers.
+## An optional log4j.properties file to overwrite the default of the Kafka brokers.
+## See an example log4j.properties at:
+## https://github.com/apache/kafka/blob/trunk/config/log4j.properties
+##
+log4j: ""
+## @param existingLog4jConfigMap The name of an existing ConfigMap containing a log4j.properties file.
+## The name of an existing ConfigMap containing a log4j.properties file.
+## NOTE: this will override log4j.
+##
+existingLog4jConfigMap: ""
+## @param heapOpts Kafka's Java Heap size
+##
+heapOpts: -Xmx1024m -Xms1024m
+## @param deleteTopicEnable Switch to enable topic deletion or not
+##
+deleteTopicEnable: {{ kafka_delete_topic_enable }}
+## @param autoCreateTopicsEnable Switch to enable auto creation of topics. Enabling auto creation of topics not recommended for production or similar environments
+##
+autoCreateTopicsEnable: true
+## @param logFlushIntervalMessages The number of messages to accept before forcing a flush of data to disk
+##
+logFlushIntervalMessages: _10000
+## @param logFlushIntervalMs The maximum amount of time a message can sit in a log before we force a flush
+##
+logFlushIntervalMs: 1000
+## @param logRetentionBytes A size-based retention policy for logs
+##
+logRetentionBytes: _1073741824
+## @param logRetentionCheckIntervalMs The interval at which log segments are checked to see if they can be deleted
+##
+logRetentionCheckIntervalMs: 300000
+## @param logRetentionHours The minimum age of a log file to be eligible for deletion due to age
+##
+logRetentionHours: 168
+## @param logSegmentBytes The maximum size of a log segment file. When this size is reached a new log segment will be created
+##
+logSegmentBytes: _1073741824
+## @param logsDirs A comma separated list of directories under which to store log files
+##
+logsDirs: /bitnami/kafka/data
+## @param maxMessageBytes The largest record batch size allowed by Kafka
+##
+maxMessageBytes: _1000012
+## @param defaultReplicationFactor Default replication factors for automatically created topics
+##
+defaultReplicationFactor: 1
+## @param offsetsTopicReplicationFactor The replication factor for the offsets topic
+##
+offsetsTopicReplicationFactor: 1
+## @param transactionStateLogReplicationFactor The replication factor for the transaction topic
+##
+transactionStateLogReplicationFactor: 1
+## @param transactionStateLogMinIsr Overridden min.insync.replicas config for the transaction topic
+##
+transactionStateLogMinIsr: 1
+## @param numIoThreads The number of threads doing disk I/O
+##
+numIoThreads: 8
+## @param numNetworkThreads The number of threads handling network requests
+##
+numNetworkThreads: 3
+## @param numPartitions The default number of log partitions per topic
+##
+numPartitions: 1
+## @param numRecoveryThreadsPerDataDir The number of threads per data directory to be used for log recovery at startup and flushing at shutdown
+##
+numRecoveryThreadsPerDataDir: 1
+## @param socketReceiveBufferBytes The receive buffer (SO_RCVBUF) used by the socket server
+##
+socketReceiveBufferBytes: 102400
+## @param socketRequestMaxBytes The maximum size of a request that the socket server will accept (protection against OOM)
+##
+socketRequestMaxBytes: _104857600
+## @param socketSendBufferBytes The send buffer (SO_SNDBUF) used by the socket server
+##
+socketSendBufferBytes: 102400
+## @param zookeeperConnectionTimeoutMs Timeout in ms for connecting to Zookeeper
+##
+zookeeperConnectionTimeoutMs: 6000
+## @param authorizerClassName The Authorizer is configured by setting authorizer.class.name=kafka.security.authorizer.AclAuthorizer in server.properties.
+##
+authorizerClassName: ""
+## @param allowEveryoneIfNoAclFound By default, if a resource has no associated ACLs, then no one is allowed to access that resource except super users.
+##
+allowEveryoneIfNoAclFound: true
+## @param superUsers You can add super users in server.properties
+##
+superUsers: User:admin
+## @param command Override kafka container command
+##
+command:
+  - /scripts/setup.sh
+## @param args Override kafka container arguments
+##
+args: []
+## @param extraEnvVars Extra environment variables to add to kafka pods (see [below]({KEY}
+## ref: https://github.com/bitnami/bitnami-docker-kafka#configuration
+## Example:
+## extraEnvVars:
+##   - name: KAFKA_CFG_BACKGROUND_THREADS
+##     value: "10"
+##
+extraEnvVars: []
+## @param extraVolumes Extra volume(s) to add to Kafka statefulset
+## Examples:
+## extraVolumes:
+##   - name: kafka-jaas
+##     secret:
+##       secretName: kafka-jaas
+extraVolumes: []
+## @param extraVolumeMounts Extra volumeMount(s) to add to Kafka containers
+## extraVolumeMounts:
+##   - name: kafka-jaas
+##     mountPath: /bitnami/kafka/config/kafka_jaas.conf
+##     subPath: kafka_jaas.conf
+extraVolumeMounts: []
+## Authentication parameteres
+## https://github.com/bitnami/bitnami-docker-kafka#security
+##
+auth:
+  ## Authentication protocol for client and inter-broker communications
+  ## This table shows the security provided on each protocol:
+  ## | Method    | Authentication                | Encryption via TLS |
+  ## | plaintext | None                          | No                 |
+  ## | tls       | None                          | Yes                |
+  ## | mtls      | Yes (two-way authentication)  | Yes                |
+  ## | sasl      | Yes (via SASL)                | No                 |
+  ## | sasl_tls  | Yes (via SASL)                | Yes                |
+  ## @param auth.clientProtocol Authentication protocol for communications with clients. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls`
+  ## @param auth.interBrokerProtocol Authentication protocol for inter-broker communications. Allowed protocols: `plaintext`, `tls`, `mtls`, `sasl` and `sasl_tls`
+  ##
+  clientProtocol: plaintext
+  interBrokerProtocol: plaintext
+  ## SASL configuration
+  ##
+  sasl:
+    ## @param auth.sasl.mechanisms SASL mechanisms when either `auth.interBrokerProtocol` or `auth.clientProtocol` are `sasl`. Allowed types: `plain`, `scram-sha-256`, `scram-sha-512`
+    ##
+    mechanisms: plain,scram-sha-256,scram-sha-512
+    ## @param auth.sasl.interBrokerMechanism SASL mechanism for inter broker communication.
+    ##
+    interBrokerMechanism: plain
+    ## JAAS configuration for SASL authentication.
+    ##
+    jaas:
+      ## @param auth.sasl.jaas.clientUsers Kafka client user list
+      ##
+      ## clientUsers:
+      ##   - user1
+      ##   - user2
+      ##
+      clientUsers:
+        - user
+      ## @param auth.sasl.jaas.clientPasswords Kafka client passwords. This is mandatory if more than one user is specified in clientUsers
+      ##
+      ## clientPasswords:
+      ##   - password1
+      ##   - password2"
+      ##
+      clientPasswords: []
+      ## @param auth.sasl.jaas.interBrokerUser Kafka inter broker communication user for SASL authentication
+      ##
+      interBrokerUser: admin
+      ## @param auth.sasl.jaas.interBrokerPassword Kafka inter broker communication password for SASL authentication
+      ##
+      interBrokerPassword: ""
+      ## @param auth.sasl.jaas.zookeeperUser Kafka Zookeeper user for SASL authentication
+      ##
+      zookeeperUser: ""
+      ## @param auth.sasl.jaas.zookeeperPassword Kafka Zookeeper password for SASL authentication
+      ##
+      zookeeperPassword: ""
+      ## @param auth.sasl.jaas.existingSecret Name of the existing secret containing credentials for clientUsers, interBrokerUser and zookeeperUser
+      ## Create this secret running the command below where SECRET_NAME is the name of the secret you want to create:
+      ##       kubectl create secret generic SECRET_NAME --from-literal=client-passwords=CLIENT_PASSWORD1,CLIENT_PASSWORD2 --from-literal=inter-broker-password=INTER_BROKER_PASSWORD --from-literal=zookeeper-password=ZOOKEEPER_PASSWORD
+      ##
+      existingSecret: ""
+  ## @param auth.saslMechanisms DEPRECATED: use `auth.sasl.mechanisms` instead.
+  ##
+  saslMechanisms: plain,scram-sha-256,scram-sha-512
+  ## @param auth.saslInterBrokerMechanism DEPRECATED: use `auth.sasl.interBrokerMechanism` instead.
+  ##
+  saslInterBrokerMechanism: plain
+  ## @param auth.jaas [object] DEPRECATED: use `auth.sasl.jaas` instead.
+  ## @skip auth.jaas.clientUsers
+  ##
+  jaas:
+    clientUsers:
+      - user
+    clientPasswords: []
+    interBrokerUser: admin
+    interBrokerPassword: ""
+    zookeeperUser: ""
+    zookeeperPassword: ""
+    existingSecret: ""
+  ## TLS configuration
+  ##
+  tls:
+    ## @param auth.tls.type Format to use for TLS certificates. Allowed types: `jks` and `pem`
+    ##
+    type: jks
+    ## @param auth.tls.existingSecrets Array existing secrets containing the TLS certificates for the Kafka brokers
+    ## When using 'jks' format for certificates, each secret should contain a truststore and a keystore.
+    ## Create these secrets following the steps below:
+    ## 1) Generate your truststore and keystore files. Helpful script: https://raw.githubusercontent.com/confluentinc/confluent-platform-security-tools/master/kafka-generate-ssl.sh
+    ## 2) Rename your truststore to `kafka.truststore.jks`.
+    ## 3) Rename your keystores to `kafka-X.keystore.jks` where X is the ID of each Kafka broker.
+    ## 4) Run the command below one time per broker to create its associated secret (SECRET_NAME_X is the name of the secret you want to create):
+    ##       kubectl create secret generic SECRET_NAME_0 --from-file=kafka.truststore.jks=./kafka.truststore.jks --from-file=kafka.keystore.jks=./kafka-0.keystore.jks
+    ##       kubectl create secret generic SECRET_NAME_1 --from-file=kafka.truststore.jks=./kafka.truststore.jks --from-file=kafka.keystore.jks=./kafka-1.keystore.jks
+    ##       ...
+    ##
+    ## When using 'pem' format for certificates, each secret should contain a public CA certificate, a public certificate and one private key.
+    ## Create these secrets following the steps below:
+    ## 1) Create a certificate key and signing request per Kafka broker, and sign the signing request with your CA
+    ## 2) Rename your CA file to `kafka.ca.crt`.
+    ## 3) Rename your certificates to `kafka-X.tls.crt` where X is the ID of each Kafka broker.
+    ## 3) Rename your keys to `kafka-X.tls.key` where X is the ID of each Kafka broker.
+    ## 4) Run the command below one time per broker to create its associated secret (SECRET_NAME_X is the name of the secret you want to create):
+    ##       kubectl create secret generic SECRET_NAME_0 --from-file=ca.crt=./kafka.ca.crt --from-file=tls.crt=./kafka-0.tls.crt --from-file=tls.key=./kafka-0.tls.key
+    ##       kubectl create secret generic SECRET_NAME_1 --from-file=ca.crt=./kafka.ca.crt --from-file=tls.crt=./kafka-1.tls.crt --from-file=tls.key=./kafka-1.tls.key
+    ##       ...
+    ##
+    existingSecrets: []
+    ## @param auth.tls.existingSecret DEPRECATED: use `auth.tls.existingSecrets` instead.
+    ##
+    existingSecret: ""
+    ## @param auth.tls.autoGenerated Generate automatically self-signed TLS certificates for Kafka brokers. Currently only supported if `auth.tls.type` is `pem`
+    ## Note: ignored when using 'jks' format or `auth.tls.existingSecrets` is not empty
+    ##
+    autoGenerated: false
+    ## @param auth.tls.password Password to access the JKS files or PEM key when they are password-protected.
+    ##
+    password: ""
+    ## @param auth.tls.jksTruststoreSecret Name of the existing secret containing your truststore if truststore not existing or different from the ones in the `auth.tls.existingSecrets`
+    ## Note: ignored when using 'pem' format for certificates .
+    ##
+    jksTruststoreSecret: ""
+    ## @param auth.tls.jksKeystoreSAN The secret key from the `auth.tls.existingSecret` containing the keystore with a SAN certificate
+    ## The SAN certificate in it should be issued with Subject Alternative Names for all headless services:
+    ##  - kafka-0.kafka-headless.kafka.svc.cluster.local
+    ##  - kafka-1.kafka-headless.kafka.svc.cluster.local
+    ##  - kafka-2.kafka-headless.kafka.svc.cluster.local
+    ## Note: ignored when using 'pem' format for certificates.
+    ##
+    jksKeystoreSAN: ""
+    ## @param auth.tls.jksTruststore The secret key from the `auth.tls.existingSecret` or `auth.tls.jksTruststoreSecret` containing the truststore
+    ## Note: ignored when using 'pem' format for certificates.
+    ##
+    jksTruststore: ""
+    ## @param auth.tls.endpointIdentificationAlgorithm The endpoint identification algorithm to validate server hostname using server certificate
+    ## Disable server host name verification by setting it to an empty string.
+    ## ref: https://docs.confluent.io/current/kafka/authentication_ssl.html#optional-settings
+    ##
+    endpointIdentificationAlgorithm: https
+  ## @param auth.jksSecret DEPRECATED: use `auth.tls.existingSecrets` instead.
+  ##
+  jksSecret: ""
+  ## @param auth.jksTruststoreSecret DEPRECATED: use `auth.tls.jksTruststoreSecret` instead.
+  ##
+  jksTruststoreSecret: ""
+  ## @param auth.jksKeystoreSAN DEPRECATED: use `auth.tls.jksKeystoreSAN` instead.
+  ##
+  jksKeystoreSAN: ""
+  ## @param auth.jksTruststore DEPRECATED: use `auth.tls.jksTruststore` instead.
+  ##
+  jksTruststore: ""
+  ## @param auth.jksPassword DEPRECATED: use `auth.tls.password` instead.
+  ##
+  jksPassword: ""
+  ## @param auth.tlsEndpointIdentificationAlgorithm DEPRECATED: use `auth.tls.endpointIdentificationAlgorithm` instead.
+  ##
+  tlsEndpointIdentificationAlgorithm: https
+## @param listeners The address(es) the socket server listens on. Auto-calculated it's set to an empty array
+## When it's set to an empty array, the listeners will be configured
+## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters)
+##
+listeners: []
+## @param advertisedListeners The address(es) (hostname:port) the broker will advertise to producers and consumers. Auto-calculated it's set to an empty array
+## When it's set to an empty array, the advertised listeners will be configured
+## based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters)
+##
+advertisedListeners: []
+## @param listenerSecurityProtocolMap The protocol->listener mapping. Auto-calculated it's set to nil
+## When it's nil, the listeners will be configured based on the authentication protocols (auth.clientProtocol and auth.interBrokerProtocol parameters)
+##
+listenerSecurityProtocolMap: ""
+## @param allowPlaintextListener Allow to use the PLAINTEXT listener
+##
+allowPlaintextListener: true
+## @param interBrokerListenerName The listener that the brokers should communicate on
+##
+interBrokerListenerName: INTERNAL
+
+## @section Statefulset parameters
+
+## @param replicaCount Number of Kafka nodes
+##
+replicaCount: {{ kafka_replica_count }}
+## @param minBrokerId Minimal broker.id value, nodes increment their `broker.id` respectively
+## Brokers increment their ID starting at this minimal value.
+## E.g., with `minBrokerId=100` and 3 nodes, IDs will be 100, 101, 102 for brokers 0, 1, and 2, respectively.
+##
+minBrokerId: 0
+## @param updateStrategy Update strategy for the stateful set
+## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets
+##
+updateStrategy: RollingUpdate
+## @param rollingUpdatePartition Partition update strategy
+## https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#partitions
+##
+rollingUpdatePartition: ""
+## @param hostAliases Add deployment host aliases
+## https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/
+##
+hostAliases: []
+## @param podManagementPolicy StatefulSet controller supports relax its ordering guarantees while preserving its uniqueness and identity guarantees. There are two valid pod management policies: OrderedReady and Parallel
+## ref: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#pod-management-policy
+##
+podManagementPolicy: Parallel
+## @param schedulerName Name of the k8s scheduler (other than default)
+## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
+##
+schedulerName: ""
+## @param podLabels Kafka pod labels
+## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+##
+podLabels: {}
+## @param podAnnotations Kafka Pod annotations
+## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+##
+podAnnotations: {}
+## @param priorityClassName Name of the existing priority class to be used by kafka pods
+## Ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/
+##
+priorityClassName: ""
+## @param podAffinityPreset Pod affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard`
+## ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
+##
+podAffinityPreset: ""
+## @param podAntiAffinityPreset Pod anti-affinity preset. Ignored if `affinity` is set. Allowed values: `soft` or `hard`
+## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity
+##
+podAntiAffinityPreset: soft
+## Node affinity preset
+## Ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity
+##
+nodeAffinityPreset:
+  ## @param nodeAffinityPreset.type Node affinity preset type. Ignored if `affinity` is set. Allowed values: `soft` or `hard`
+  ##
+  type: ""
+  ## @param nodeAffinityPreset.key Node label key to match Ignored if `affinity` is set.
+  ## E.g.
+  ## key: "kubernetes.io/e2e-az-name"
+  ##
+  key: ""
+  ## @param nodeAffinityPreset.values Node label values to match. Ignored if `affinity` is set.
+  ## E.g.
+  ## values:
+  ##   - e2e-az1
+  ##   - e2e-az2
+  ##
+  values: []
+## @param affinity Affinity for pod assignment
+## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+## Note: podAffinityPreset, podAntiAffinityPreset, and  nodeAffinityPreset will be ignored when it's set
+##
+affinity: {}
+## @param nodeSelector Node labels for pod assignment
+## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+##
+nodeSelector: {}
+## @param tolerations Tolerations for pod assignment
+## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+##
+tolerations: []
+## @param topologySpreadConstraints Topology Spread Constraints for pod assignment spread across your cluster among failure-domains. Evaluated as a template
+## Ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/#spread-constraints-for-pods
+##
+topologySpreadConstraints: {}
+## @param terminationGracePeriodSeconds Seconds the pod needs to gracefully terminate
+## ref: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#hook-handler-execution
+##
+terminationGracePeriodSeconds: ""
+## Kafka pods' Security Context
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod
+## @param podSecurityContext.enabled Enable security context for the pods
+## @param podSecurityContext.fsGroup Group ID for the filesystem used by the containers
+## @param podSecurityContext.runAsUser User ID for the service user running the pod
+##
+podSecurityContext:
+  enabled: true
+  fsGroup: 1001
+  runAsUser: 1001
+## @param containerSecurityContext Kafka containers' Security Context
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container
+## Example:
+##   containerSecurityContext:
+##     capabilities:
+##       drop: ["NET_RAW"]
+##     readOnlyRootFilesystem: true
+##
+containerSecurityContext: {}
+## Kafka containers' resource requests and limits
+## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+## We usually recommend not to specify default resources and to leave this as a conscious
+## choice for the user. This also increases chances charts run on environments with little
+## resources, such as Minikube. If you do want to specify resources, uncomment the following
+## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+## @param resources.limits The resources limits for Kafka containers
+## @param resources.requests The requested resources for Kafka containers
+##
+resources:
+  ## Example:
+  ## limits:
+  ##    cpu: 250m
+  ##    memory: 1Gi
+  limits: {}
+  ## Examples:
+  ## requests:
+  ##    cpu: 250m
+  ##    memory: 256Mi
+  requests: {}
+## Kafka containers' liveness probe. Evaluated as a template.
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
+## @param livenessProbe.enabled Enable livenessProbe
+## @param livenessProbe.initialDelaySeconds Initial delay seconds for livenessProbe
+## @param livenessProbe.periodSeconds Period seconds for livenessProbe
+## @param livenessProbe.timeoutSeconds Timeout seconds for livenessProbe
+## @param livenessProbe.failureThreshold Failure threshold for livenessProbe
+## @param livenessProbe.successThreshold Success threshold for livenessProbe
+##
+livenessProbe:
+  enabled: true
+  initialDelaySeconds: 10
+  timeoutSeconds: 5
+  failureThreshold: 3
+  periodSeconds: 10
+  successThreshold: 1
+## Kafka containers' readiness probe. Evaluated as a template.
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#container-probes
+## @param readinessProbe.enabled Enable readinessProbe
+## @param readinessProbe.initialDelaySeconds Initial delay seconds for readinessProbe
+## @param readinessProbe.periodSeconds Period seconds for readinessProbe
+## @param readinessProbe.timeoutSeconds Timeout seconds for readinessProbe
+## @param readinessProbe.failureThreshold Failure threshold for readinessProbe
+## @param readinessProbe.successThreshold Success threshold for readinessProbe
+##
+readinessProbe:
+  enabled: true
+  initialDelaySeconds: 5
+  failureThreshold: 6
+  timeoutSeconds: 5
+  periodSeconds: 10
+  successThreshold: 1
+## @param customLivenessProbe Custom Liveness probe configuration for Kafka
+##
+customLivenessProbe: {}
+## @param customReadinessProbe Custom Readiness probe configuration for Kafka
+##
+customReadinessProbe: {}
+## Pod Disruption Budget configuration
+## The PDB will only be created if replicaCount is greater than 1
+## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions
+##
+pdb:
+  ## @param pdb.create Enable/disable a Pod Disruption Budget creation
+  ##
+  create: false
+  ## @param pdb.minAvailable Minimum number/percentage of pods that should remain scheduled
+  ##
+  minAvailable: ""
+  ## @param pdb.maxUnavailable Maximum number/percentage of pods that may be made unavailable
+  ##
+  maxUnavailable: 1
+## @param sidecars Attach additional sidecar containers to the Kafka pod
+## Example:
+## sidecars:
+##   - name: your-image-name
+##     image: your-image
+##     imagePullPolicy: Always
+##     ports:
+##       - name: portname
+##         containerPort: 1234
+##
+sidecars: []
+## @param initContainers Add extra init containers
+##
+initContainers: []
+
+## @section Exposure parameters
+
+## Service parameters
+##
+service:
+  ## @param service.type Kubernetes Service type
+  ##
+  type: ClusterIP
+  ## @param service.port Kafka port for client connections
+  ##
+  port: 9092
+  ## @param service.internalPort Kafka port for inter-broker connections
+  ##
+  internalPort: 9093
+  ## @param service.externalPort Kafka port for external connections
+  ##
+  externalPort: 9094
+  ## @param service.nodePorts [object] Specify the nodePort value for the LoadBalancer and NodePort service types.
+  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+  ##
+  nodePorts:
+    client: ""
+    external: ""
+  ## @param service.loadBalancerIP loadBalancerIP for Kafka Service
+  ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
+  ##
+  loadBalancerIP: ""
+  ## @param service.loadBalancerSourceRanges Address(es) that are allowed when service is LoadBalancer
+  ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service
+  ## Example:
+  ## loadBalancerSourceRanges:
+  ## - 10.10.10.0/24
+  ##
+  loadBalancerSourceRanges: []
+  ## @param service.annotations Service annotations
+  ##
+  annotations: {}
+## External Access to Kafka brokers configuration
+##
+externalAccess:
+  ## @param externalAccess.enabled Enable Kubernetes external cluster access to Kafka brokers
+  ##
+  enabled: true
+  ## External IPs auto-discovery configuration
+  ## An init container is used to auto-detect LB IPs or node ports by querying the K8s API
+  ## Note: RBAC might be required
+  ##
+  autoDiscovery:
+    ## @param externalAccess.autoDiscovery.enabled Enable using an init container to auto-detect external IPs/ports by querying the K8s API
+    ##
+    enabled: true
+    ## Bitnami Kubectl image
+    ## ref: https://hub.docker.com/r/bitnami/kubectl/tags/
+    ## @param externalAccess.autoDiscovery.image.registry Init container auto-discovery image registry
+    ## @param externalAccess.autoDiscovery.image.repository Init container auto-discovery image repository
+    ## @param externalAccess.autoDiscovery.image.tag Init container auto-discovery image tag (immutable tags are recommended)
+    ## @param externalAccess.autoDiscovery.image.pullPolicy Init container auto-discovery image pull policy
+    ## @param externalAccess.autoDiscovery.image.pullSecrets Init container auto-discovery image pull secrets
+    ##
+    image:
+      registry: docker.io
+      repository: bitnami/kubectl
+      tag: 1.19.15-debian-10-r39
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace)
+      ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    ## Init Container resource requests and limits
+    ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+    ## We usually recommend not to specify default resources and to leave this as a conscious
+    ## choice for the user. This also increases chances charts run on environments with little
+    ## resources, such as Minikube. If you do want to specify resources, uncomment the following
+    ## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    ## @param externalAccess.autoDiscovery.resources.limits Init container auto-discovery resource limits
+    ## @param externalAccess.autoDiscovery.resources.requests Init container auto-discovery resource requests
+    ##
+    resources:
+      ## Example:
+      ## limits:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      limits: {}
+      ## Examples:
+      ## requests:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      requests: {}
+  ## Parameters to configure K8s service(s) used to externally access Kafka brokers
+  ## A new service per broker will be created
+  ##
+  service:
+    ## @param externalAccess.service.type Kubernetes Service type for external access. It can be NodePort or LoadBalancer
+    ##
+    type: "{{ service_type }}"
+    ## @param externalAccess.service.port Kafka port used for external access when service type is LoadBalancer
+    ##
+    port: {{ service_port }}
+    ## @param externalAccess.service.loadBalancerIPs Array of load balancer IPs for each Kafka broker. Length must be the same as replicaCount
+    ## Example:
+    ## loadBalancerIPs:
+    ##   - X.X.X.X
+    ##   - Y.Y.Y.Y
+    ##
+    loadBalancerIPs: []
+    ## @param externalAccess.service.loadBalancerSourceRanges Address(es) that are allowed when service is LoadBalancer
+    ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service
+    ## Example:
+    ## loadBalancerSourceRanges:
+    ## - 10.10.10.0/24
+    ##
+    loadBalancerSourceRanges: []
+    ## @param externalAccess.service.nodePorts Array of node ports used for each Kafka broker. Length must be the same as replicaCount
+    ## Example:
+    ## nodePorts:
+    ##   - 30001
+    ##   - 30002
+    ##
+    nodePorts: []
+    ## @param externalAccess.service.useHostIPs Use service host IPs to configure Kafka external listener when service type is NodePort
+    ##
+    useHostIPs: false
+    ## @param externalAccess.service.domain Domain or external ip used to configure Kafka external listener when service type is NodePort
+    ## If not specified, the container will try to get the kubernetes node external IP
+    ##
+    domain: ""
+    ## @param externalAccess.service.annotations Service annotations for external access
+    ##
+    annotations: {}
+    ## @param externalAccess.service.usePodIPs using the MY_POD_IP address for external access.
+    ##
+    usePodIPs: false
+
+## @section Persistence parameters
+
+## Persistence parameters
+##
+persistence:
+  ## @param persistence.enabled Enable Kafka data persistence using PVC, note that Zookeeper persistence is unaffected
+  ##
+  enabled: true
+  ## @param persistence.existingClaim Provide an existing `PersistentVolumeClaim`, the value is evaluated as a template
+  ## If defined, PVC must be created manually before volume will be bound
+  ## The value is evaluated as a template
+  ##
+  existingClaim: ""
+  ## @param persistence.storageClass PVC Storage Class for Kafka data volume
+  ## If defined, storageClassName: <storageClass>
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ## set, choosing the default provisioner.
+  ##
+  storageClass: ""
+  ## @param persistence.accessModes PV Access Mode
+  ##
+  accessModes:
+    - ReadWriteOnce
+  ## @param persistence.size PVC Storage Request for Kafka data volume
+  ##
+  size: {{ kafka_persistence_size }}
+  ## @param persistence.annotations Annotations for the PVC
+  ##
+  annotations: {}
+  ## @param persistence.selector Selector to match an existing Persistent Volume for Kafka's data PVC. If set, the PVC can't have a PV dynamically provisioned for it
+  ## selector:
+  ##   matchLabels:
+  ##     app: my-app
+  selector: {}
+  ## @param persistence.mountPath Mount path of the Kafka data volume
+  ##
+  mountPath: /bitnami/kafka
+## Log Persistence parameters
+##
+logPersistence:
+  ## @param logPersistence.enabled Enable Kafka logs persistence using PVC, note that Zookeeper persistence is unaffected
+  ##
+  enabled: false
+  ## @param logPersistence.existingClaim A manually managed Persistent Volume and Claim
+  ## If defined, PVC must be created manually before volume will be bound
+  ## The value is evaluated as a template
+  ##
+  existingClaim: ""
+  ## @param logPersistence.existingLogClaim PV Storage Class
+  ## If defined, storageClassName: <storageClass>
+  ## If set to "-", storageClassName: "", which disables dynamic provisioning
+  ## If undefined (the default) or set to null, no storageClassName spec is
+  ## set, choosing the default provisioner.
+  existingLogClaim: ""
+  ## @param logPersistence.accessModes PV Access Mode
+  ##
+  accessModes:
+    - ReadWriteOnce
+  ## @param logPersistence.size PVC Storage Request for Kafka logs volume
+  ##
+  size: 8Gi
+  ## @param logPersistence.annotations Annotations for the PVC
+  ##
+  annotations: {}
+  ## @param logPersistence.selector Selector to match an existing Persistent Volume for Kafka's log data PVC. If set, the PVC can't have a PV dynamically provisioned for it
+  ## selector:
+  ##   matchLabels:
+  ##     app: my-app
+  selector: {}
+  ## @param logPersistence.mountPath Mount path of the Kafka logs volume
+  ##
+  mountPath: /opt/bitnami/kafka/logs
+
+## @section RBAC parameters
+
+## Kafka pods ServiceAccount
+## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+##
+serviceAccount:
+  ## @param serviceAccount.create Enable creation of ServiceAccount for Kafka pods
+  ##
+  create: true
+  ## @param serviceAccount.name The name of the service account to use. If not set and `create` is `true`, a name is generated
+  ## If not set and create is true, a name is generated using the kafka.serviceAccountName template
+  ##
+  name: ""
+  ## @param serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created
+  ## Can be set to false if pods using this serviceAccount do not need to use K8s API
+  ##
+  automountServiceAccountToken: true
+## Role Based Access
+## ref: https://kubernetes.io/docs/admin/authorization/rbac/
+##
+rbac:
+  ## @param rbac.create Whether to create & use RBAC resources or not
+  ## binding Kafka ServiceAccount to a role
+  ## that allows Kafka pods querying the K8s API
+  ##
+  create: true
+
+## @section Volume Permissions parameters
+
+## Init Container parameters
+## Change the owner and group of the persistent volume(s) mountpoint(s) to 'runAsUser:fsGroup' on each component
+## values from the securityContext section of the component
+##
+volumePermissions:
+  ## @param volumePermissions.enabled Enable init container that changes the owner and group of the persistent volume(s) mountpoint to `runAsUser:fsGroup`
+  ##
+  enabled: false
+  ## The security context for the volumePermissions init container
+  ## @param volumePermissions.securityContext.runAsUser User ID for the container
+  ##
+  securityContext:
+    runAsUser: 0
+  ## @param volumePermissions.image.registry Init container volume-permissions image registry
+  ## @param volumePermissions.image.repository Init container volume-permissions image name
+  ## @param volumePermissions.image.tag Init container volume-permissions image tag
+  ## @param volumePermissions.image.pullPolicy Init container volume-permissions image pull policy
+  ## @param volumePermissions.image.pullSecrets Specify docker-registry secret names as an array
+  ##
+  image:
+    registry: docker.io
+    repository: bitnami/bitnami-shell
+    tag: 10-debian-10-r234
+    ## Specify a imagePullPolicy
+    ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+    ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+    ##
+    pullPolicy: IfNotPresent
+    ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace)
+    ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+    ## Example:
+    ## pullSecrets:
+    ##   - myRegistryKeySecretName
+    ##
+    pullSecrets: []
+  ## Init Container resource requests and limits
+  ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+  ## We usually recommend not to specify default resources and to leave this as a conscious
+  ## choice for the user. This also increases chances charts run on environments with little
+  ## resources, such as Minikube. If you do want to specify resources, uncomment the following
+  ## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  ## @param volumePermissions.resources.limits Init container volume-permissions resource  limits
+  ## @param volumePermissions.resources.requests Init container volume-permissions resource  requests
+  ##
+  resources:
+    ## Example:
+    ## limits:
+    ##    cpu: 100m
+    ##    memory: 128Mi
+    limits: {}
+    ## Examples:
+    ## requests:
+    ##    cpu: 100m
+    ##    memory: 128Mi
+    requests: {}
+
+## @section Metrics parameters
+
+## Prometheus Exporters / Metrics
+##
+metrics:
+  ## Prometheus Kafka Exporter: exposes complimentary metrics to JMX Exporter
+  ##
+  kafka:
+    ## @param metrics.kafka.enabled Whether or not to create a standalone Kafka exporter to expose Kafka metrics
+    ##
+    enabled: false
+    ## Bitnami Kafka exporter image
+    ## ref: https://hub.docker.com/r/bitnami/kafka-exporter/tags/
+    ## @param metrics.kafka.image.registry Kafka exporter image registry
+    ## @param metrics.kafka.image.repository Kafka exporter image repository
+    ## @param metrics.kafka.image.tag Kafka exporter image tag (immutable tags are recommended)
+    ## @param metrics.kafka.image.pullPolicy Kafka exporter image pull policy
+    ## @param metrics.kafka.image.pullSecrets Specify docker-registry secret names as an array
+    ##
+    image:
+      registry: docker.io
+      repository: bitnami/kafka-exporter
+      tag: 1.4.2-debian-10-r41
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace)
+      ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    ## Kafka exporter pods ServiceAccount
+    ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/
+    ##
+    serviceAccount:
+      ## @param metrics.kafka.serviceAccount.create Enable creation of ServiceAccount for Kafka exporter pods
+      ##
+      create: true
+      ## @param metrics.kafka.serviceAccount.name The name of the service account to use. If not set and `create` is `true`, a name is generated
+      ## If not set and create is true, a name is generated using the kafka.metrics.kafka.serviceAccountName template
+      ##
+      name: ""
+      ## @param metrics.kafka.serviceAccount.automountServiceAccountToken Allows auto mount of ServiceAccountToken on the serviceAccount created
+      ## Can be set to false if pods using this serviceAccount do not need to use K8s API
+      ##
+      automountServiceAccountToken: true
+    ## @param metrics.kafka.schedulerName Name of the k8s scheduler (other than default) for Kafka Exporter
+    ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
+    ##
+    schedulerName: ""
+    ## @param metrics.kafka.extraFlags Extra flags to be passed to Kafka exporter
+    ## Example:
+    ## extraFlags:
+    ##   tls.insecure-skip-tls-verify: ""
+    ##   web.telemetry-path: "/metrics"
+    ##
+    extraFlags: {}
+    ## @param metrics.kafka.certificatesSecret Name of the existing secret containing the optional certificate and key files
+    ## for Kafka Exporter client authentication
+    ##
+    certificatesSecret: ""
+    ## @param metrics.kafka.tlsCert The secret key from the certificatesSecret if 'client-cert' key different from the default (cert-file)
+    ##
+    tlsCert: cert-file
+    ## @param metrics.kafka.tlsKey The secret key from the certificatesSecret if 'client-key' key different from the default (key-file)
+    ##
+    tlsKey: key-file
+    ## @param metrics.kafka.tlsCaSecret Name of the existing secret containing the optional ca certificate for Kafka Exporter client authentication
+    ##
+    tlsCaSecret: ""
+    ## @param metrics.kafka.tlsCaCert The secret key from the certificatesSecret or tlsCaSecret if 'ca-cert' key different from the default (ca-file)
+    ##
+    tlsCaCert: ca-file
+    ## @param metrics.kafka.podLabels Kafka exporter pod labels
+    ## Ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+    ##
+    podLabels: {}
+    ## @param metrics.kafka.podAnnotations Kafka exporter pod annotations
+    ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/
+    ##
+    podAnnotations: {}
+    ## Prometheus Kafka Exporter' resource requests and limits
+    ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+    ## We usually recommend not to specify default resources and to leave this as a conscious
+    ## choice for the user. This also increases chances charts run on environments with little
+    ## resources, such as Minikube. If you do want to specify resources, uncomment the following
+    ## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    ## @param metrics.kafka.resources.limits Kafka Exporter container resource limits
+    ## @param metrics.kafka.resources.requests Kafka Exporter container resource requests
+    ##
+    resources:
+      ## Example:
+      ## limits:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      limits: {}
+      ## Examples:
+      ## requests:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      requests: {}
+    ## @param metrics.kafka.affinity Affinity for Kafka Exporter pod assignment
+    ## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+    ## Note: podAffinityPreset, podAntiAffinityPreset, and  nodeAffinityPreset will be ignored when it's set
+    ##
+    affinity: {}
+    ## @param metrics.kafka.nodeSelector Node labels for Kafka Exporter pod assignment
+    ## Ref: https://kubernetes.io/docs/user-guide/node-selection/
+    ##
+    nodeSelector: {}
+    ## @param metrics.kafka.tolerations Tolerations for Kafka Exporter pod assignment
+    ## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
+    ##
+    tolerations: []
+    ## @param metrics.kafka.initContainers Add init containers to the Kafka exporter pods
+    ## Example:
+    ## initContainers:
+    ##   - name: your-image-name
+    ##     image: your-image
+    ##     imagePullPolicy: Always
+    ##     ports:
+    ##       - name: portname
+    ##         containerPort: 1234
+    ##
+    initContainers: []
+    ## Service configuration
+    ##
+    service:
+      ## @param metrics.kafka.service.type Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for Kafka Exporter
+      ##
+      type: ClusterIP
+      ## @param metrics.kafka.service.port Kafka Exporter Prometheus port
+      ##
+      port: 9308
+      ## @param metrics.kafka.service.nodePort Kubernetes HTTP node port
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+      ##
+      nodePort: ""
+      ## @param metrics.kafka.service.loadBalancerIP loadBalancerIP if service type is `LoadBalancer`
+      ## Set the LoadBalancer service type to internal only
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
+      ##
+      loadBalancerIP: ""
+      ## @param metrics.kafka.service.loadBalancerSourceRanges Load Balancer sources
+      ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service
+      ## Example:
+      ## loadBalancerSourceRanges:
+      ## - 10.10.10.0/24
+      ##
+      loadBalancerSourceRanges: []
+      ## @param metrics.kafka.service.clusterIP Static clusterIP or None for headless services
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address
+      ##
+      clusterIP: ""
+      ## @param metrics.kafka.service.annotations [object] Annotations for the Kafka Exporter Prometheus metrics service
+      ##
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "9308"
+        prometheus.io/path: "/metrics"
+  ## Prometheus JMX Exporter: exposes the majority of Kafkas metrics
+  ##
+  jmx:
+    ## @param metrics.jmx.enabled Whether or not to expose JMX metrics to Prometheus
+    ##
+    enabled: false
+    ## Bitnami JMX exporter image
+    ## ref: https://hub.docker.com/r/bitnami/jmx-exporter/tags/
+    ## @param metrics.jmx.image.registry JMX exporter image registry
+    ## @param metrics.jmx.image.repository JMX exporter image repository
+    ## @param metrics.jmx.image.tag JMX exporter image tag (immutable tags are recommended)
+    ## @param metrics.jmx.image.pullPolicy JMX exporter image pull policy
+    ## @param metrics.jmx.image.pullSecrets Specify docker-registry secret names as an array
+    ##
+    image:
+      registry: docker.io
+      repository: bitnami/jmx-exporter
+      tag: 0.16.1-debian-10-r103
+      ## Specify a imagePullPolicy
+      ## Defaults to 'Always' if image tag is 'latest', else set to 'IfNotPresent'
+      ## ref: http://kubernetes.io/docs/user-guide/images/#pre-pulling-images
+      ##
+      pullPolicy: IfNotPresent
+      ## Optionally specify an array of imagePullSecrets (secrets must be manually created in the namespace)
+      ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
+      ## Example:
+      ## pullSecrets:
+      ##   - myRegistryKeySecretName
+      ##
+      pullSecrets: []
+    ## Prometheus JMX Exporter' resource requests and limits
+    ## ref: http://kubernetes.io/docs/user-guide/compute-resources/
+    ## We usually recommend not to specify default resources and to leave this as a conscious
+    ## choice for the user. This also increases chances charts run on environments with little
+    ## resources, such as Minikube. If you do want to specify resources, uncomment the following
+    ## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+    ## @param metrics.jmx.resources.limits JMX Exporter container resource limits
+    ## @param metrics.jmx.resources.requests JMX Exporter container resource requests
+    ##
+    resources:
+      ## Example:
+      ## limits:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      limits: {}
+      ## Examples:
+      ## requests:
+      ##    cpu: 100m
+      ##    memory: 128Mi
+      requests: {}
+    ## Service configuration
+    ##
+    service:
+      ## @param metrics.jmx.service.type Kubernetes service type (`ClusterIP`, `NodePort` or `LoadBalancer`) for JMX Exporter
+      ##
+      type: ClusterIP
+      ## @param metrics.jmx.service.port JMX Exporter Prometheus port
+      ##
+      port: 5556
+      ## @param metrics.jmx.service.nodePort Kubernetes HTTP node port
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport
+      ##
+      nodePort: ""
+      ## @param metrics.jmx.service.loadBalancerIP loadBalancerIP if service type is `LoadBalancer`
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#internal-load-balancer
+      ##
+      loadBalancerIP: ""
+      ## @param metrics.jmx.service.loadBalancerSourceRanges Load Balancer sources
+      ## ref: https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service
+      ## Example:
+      ## loadBalancerSourceRanges:
+      ## - 10.10.10.0/24
+      ##
+      loadBalancerSourceRanges: []
+      ## @param metrics.jmx.service.clusterIP Static clusterIP or None for headless services
+      ## ref: https://kubernetes.io/docs/concepts/services-networking/service/#choosing-your-own-ip-address
+      ##
+      clusterIP: ""
+      ## @param metrics.jmx.service.annotations [object] Annotations for the JMX Exporter Prometheus metrics service
+      ##
+      annotations:
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "5556"
+        prometheus.io/path: "/"
+    ## @param metrics.jmx.whitelistObjectNames Allows setting which JMX objects you want to expose to via JMX stats to JMX Exporter
+    ## Only whitelisted values will be exposed via JMX Exporter. They must also be exposed via Rules. To expose all metrics
+    ## (warning its crazy excessive and they aren't formatted in a prometheus style) (1) `whitelistObjectNames: []`
+    ## (2) commented out above `overrideConfig`.
+    ##
+    whitelistObjectNames:
+      - kafka.controller:*
+      - kafka.server:*
+      - java.lang:*
+      - kafka.network:*
+      - kafka.log:*
+    ## @param metrics.jmx.config [string] Configuration file for JMX exporter
+    ## Specify content for jmx-kafka-prometheus.yml. Evaluated as a template
+    ##
+    ## Credits to the incubator/kafka chart for the JMX configuration.
+    ## https://github.com/helm/charts/tree/master/incubator/kafka
+    ##
+    config: |-
+      jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi
+      lowercaseOutputName: true
+      lowercaseOutputLabelNames: true
+      ssl: false
+      whitelistObjectNames: ["\"kafka.controller:*\", \"kafka.server:*\", \"java.lang:*\", \"kafka.network:*\", \"kafka.log:*\""]
+    ## @param metrics.jmx.existingConfigmap Name of existing ConfigMap with JMX exporter configuration
+    ## NOTE: This will override metrics.jmx.config
+    ##
+    existingConfigmap: ""
+  ## Prometheus Operator ServiceMonitor configuration
+  ##
+  serviceMonitor:
+    ## @param metrics.serviceMonitor.enabled if `true`, creates a Prometheus Operator ServiceMonitor (requires `metrics.kafka.enabled` or `metrics.jmx.enabled` to be `true`)
+    ##
+    enabled: false
+    ## @param metrics.serviceMonitor.namespace Namespace in which Prometheus is running
+    ##
+    namespace: ""
+    ## @param metrics.serviceMonitor.interval Interval at which metrics should be scraped
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    interval: ""
+    ## @param metrics.serviceMonitor.scrapeTimeout Timeout after which the scrape is ended
+    ## ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
+    ##
+    scrapeTimeout: ""
+    ## @param metrics.serviceMonitor.selector ServiceMonitor selector labels
+    ## ref: https://github.com/bitnami/charts/tree/master/bitnami/prometheus-operator#prometheus-configuration
+    ## e.g:
+    ## selector:
+    ##   prometheus: my-prometheus
+    ##
+    selector: {}
+    ## @param metrics.serviceMonitor.relabelings Relabel configuration for the metrics
+    ##
+    relabelings: []
+    ## @param metrics.serviceMonitor.metricRelabelings MetricRelabelConfigs to apply to samples before ingestion
+    ##
+    metricRelabelings: []
+
+## @section Kafka provisioning parameters
+
+## Kafka provisioning
+##
+provisioning:
+  ## @param provisioning.enabled Enable kafka provisioning Job
+  ##
+  enabled: false
+  ## @param provisioning.numPartitions Default number of partitions for topics when unspecified.
+  numPartitions: 1
+  ## @param provisioning.replicationFactor Default replication factor for topics when unspecified.
+  replicationFactor: 1
+  ## @param provisioning.schedulerName Name of the k8s scheduler (other than default) for kafka provisioning
+  ## ref: https://kubernetes.io/docs/tasks/administer-cluster/configure-multiple-schedulers/
+  ##
+  schedulerName: ""
+  ## @param provisioning.podAnnotations Provisioning Pod annotations.
+  ##
+  podAnnotations: {}
+  ## We usually recommend not to specify default resources and to leave this as a conscious
+  ## choice for the user. This also increases chances charts run on environments with little
+  ## resources, such as Minikube. If you do want to specify resources, uncomment the following
+  ## lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  ## @param provisioning.resources.limits The resources limits for the container
+  ## @param provisioning.resources.requests The requested resources for the container
+  ##
+  resources:
+    ## Example:
+    ## limits:
+    ##    cpu: 250m
+    ##    memory: 1Gi
+    limits: {}
+    ## Examples:
+    ## requests:
+    ##    cpu: 250m
+    ##    memory: 256Mi
+    requests: {}
+  ## @param provisioning.command Override provisioning container command
+  ##
+  command: []
+  ## @param provisioning.args Override provisioning container arguments
+  ##
+  args: []
+  ## @param provisioning.topics Kafka provisioning topics
+  ## - name: topic-name
+  ##   partitions: 1
+  ##   replicationFactor: 1
+  ##   ## https://kafka.apache.org/documentation/#topicconfigs
+  ##   config:
+  ##     max.message.bytes: 64000
+  ##     flush.messages: 1
+  ##
+  topics: []
+
+## @section Zookeeper chart parameters
+
+## Zookeeper chart configuration
+## https://github.com/bitnami/charts/blob/master/bitnami/zookeeper/values.yaml
+##
+zookeeper:
+  ## @param zookeeper.enabled Switch to enable or disable the Zookeeper helm chart
+  ##
+  enabled: {{ zookeeper_enabled }}
+  auth:
+    ## @param zookeeper.auth.enabled Enable Zookeeper auth
+    ##
+    enabled: false
+    ## @param zookeeper.auth.clientUser User that will use Zookeeper clients to auth
+    ##
+    clientUser: ""
+    ## @param zookeeper.auth.clientPassword Password that will use Zookeeper clients to auth
+    ##
+    clientPassword: ""
+    ## @param zookeeper.auth.serverUsers Comma, semicolon or whitespace separated list of user to be created. Specify them as a string, for example: "user1,user2,admin"
+    ##
+    serverUsers: ""
+    ## @param zookeeper.auth.serverPasswords Comma, semicolon or whitespace separated list of passwords to assign to users when created. Specify them as a string, for example: "pass4user1, pass4user2, pass4admin"
+    ##
+    serverPasswords: ""
+  heapSize: {{ zookeeper_heapsize }}
+  replicaCount: {{ zookeeper_replica_count }} 
+
+## This value is only used when zookeeper.enabled is set to false
+##
+externalZookeeper:
+  ## @param externalZookeeper.servers Server or list of external Zookeeper servers to use
+  ##
+  servers: []
+
+#################### Zookeeper Variables ######################
+
+# zookeeper:
+#   image:
+#     registry: docker.io
+#     repository: bitnami/zookeeper
+#     tag: 3.6-debian-10
+#     pullPolicy: Always
+
+#   heapSize: 256
+#   replicaCount: 3
+
+#   resources:
+#     requests:
+#       memory: 256Mi
+#       cpu: 512m
+
+#   tickTime: 2000
+#   initLimit: 10
+#   syncLimit: 5
+#   preAllocSize: 65536
+#   snapCount: 100000
+#   maxClientCnxns: 60
+#   fourlwCommandsWhitelist: srvr, mntr, ruok
+#   listenOnAllIPs: false
+#   allowAnonymousLogin: true
+#   autopurge:
+#     snapRetainCount: 3
+#     purgeInterval: 0
+#   maxSessionTimeout: 40000
+
+#   allowAnonymousLogin: true
+
+#   minServerId: 1
+
+#   # securityContext:
+#   #   enabled: true
+#   #   fsGroup: 1001
+#   #   runAsUser: 1001
+
+#   livenessProbe:
+#     enabled: true
+#     initialDelaySeconds: 30
+#     periodSeconds: 10
+#     timeoutSeconds: 5
+#     failureThreshold: 6
+#     successThreshold: 1
+#     probeCommandTimeout: 2
+
+#   readinessProbe:
+#     enabled: true
+#     initialDelaySeconds: 5
+#     periodSeconds: 10
+#     timeoutSeconds: 5
+#     failureThreshold: 6
+#     successThreshold: 1
+#     probeCommandTimeout: 2
+
+#   service:
+#     type: ClusterIP
+#     # loadBalancerIP: ""
+#     port: 2181
+#     followerPort: 2888
+#     electionPort: 3888
+#     nodePorts:
+#       client: ""
+#       clientTls: ""
+#     publishNotReadyAddresses: true
+#     tlsClientPort: 3181
+#     disableBaseClientPort: false
+#     annotations: {}
+#     headless:
+#       annotations: {}
+
+
diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore b/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore
new file mode 100644
index 0000000000..50af031725
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml
new file mode 100644
index 0000000000..2ac719df1c
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/Chart.yaml
@@ -0,0 +1,21 @@
+apiVersion: v2
+name: additional-scrape-configs
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.16.0
diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl
new file mode 100644
index 0000000000..c66316674f
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "additional-scrape-configs.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "additional-scrape-configs.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "additional-scrape-configs.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "additional-scrape-configs.labels" -}}
+helm.sh/chart: {{ include "additional-scrape-configs.chart" . }}
+{{ include "additional-scrape-configs.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "additional-scrape-configs.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "additional-scrape-configs.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "additional-scrape-configs.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "additional-scrape-configs.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml
new file mode 100644
index 0000000000..cbc1ca2180
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/templates/secrets.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.fullnameOverride }}-prometheus-scrape-confg
+  namespace: {{ default .Values.namespace .Release.Namespace  }}
+  labels:
+    {{- include "additional-scrape-configs.labels" . | nindent 4 }}
+    app: {{ .Values.fullnameOverride }}-scrape-confg
+data:
+  additional-scrape-configs.yaml: {{ toYaml .Values.scrapeconfig | b64enc | quote }}
diff --git a/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml b/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml
new file mode 100644
index 0000000000..9d81e6af57
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/additional-scrape-configs/values.yaml
@@ -0,0 +1,10 @@
+# This value will help to scrape external configs,
+# which your prometheus have to watch
+scrapeconfig: []
+  # - job_name: 'Druid'
+  #   metrics_path: /metrics/druid
+  #   static_configs:
+  #     - targets: ["11.2.4.31:9000"]
+  #       labels:
+  #         "service_name": "druid"
+fullnameOverride: "sunbird-monitoring"
diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml
index 7374424f24..5e997514db 100644
--- a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml
+++ b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs.yaml
@@ -17,82 +17,396 @@ spec:
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) > {{ .Values.telemetry_extractor_threshold_critical }} 
+        message: The lag for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_threshold_critical | int }}
         summary: telemetry-extractor lag is critical
+        lag: {{`({{ humanize $value  }})`}}
+        job_id: telemetry-extractor
+        alertname: FlinkJobsLag
 
     - alert: pipeline-preprocessor lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_critical }} 
+        message: The lag for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_threshold_critical | int }}
         summary: pipeline-preprocessor lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: pipeline-preprocessor
+        alertname: FlinkJobsLag
    
-    - alert:  de-normalization lag critical
-      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }}
+    - alert:  de-normalization-primary lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }}
-        summary: de-normalization lag is critical
+        message: The lag for de-normalization-primary job is more than threshold {{ .Values.de_normalization_threshold_critical | int }}
+        summary: de-normalization-primary lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-primary
+        alertname: FlinkJobsLag
+
+    - alert:  de-normalization-secondary lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_threshold_critical | int }}
+        summary: de-normalization-secondary lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-secondary
+        alertname: FlinkJobsLag
 
     - alert:  druid-validator lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "drud-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_critical }}
+        message: The lag for druid-validator job is more than threshold {{ .Values.druid_validator_threshold_critical | int }}
         summary: druid-validator lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: druid-validator
+        alertname: FlinkJobsLag
 
     - alert: assessment-aggregator lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_critical }}
+        message: The lag for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_threshold_critical | int }}
         summary: assessment-aggregator lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: assessment-aggregator
+        alertname: FlinkJobsLag
 
     - alert: content-cache-updater lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_critical }}
+        message: The lag for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_threshold_critical | int }}
         summary: content-cache-updater lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: content-cache-updater
+        alertname: FlinkJobsLag
 
     - alert: user-cache-updater lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical }}
+        message: The lag for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_threshold_critical | int }}
         summary: user-cache-updater lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater
+        alertname: FlinkJobsLag
 
     - alert: summary-denormalization lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_critical }}
+        message: The lag for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_threshold_critical | int }}
         summary: summary-denormalization lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: summary-denormalization
+        alertname: FlinkJobsLag
 
     - alert: device-profile-updater lag critical
       expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values.device_profile_updater_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values._device_profile_updater_threshold_critical }}
+        message: The lag for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_threshold_critical | int }}
         summary: device-profile-updater lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: device-profile-updater
+        alertname: FlinkJobsLag
+
+    - alert: user-cache-updater-v2 lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_critical}}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_updater_threshold_critical | int }}
+        summary: user-cache-updater-v2 lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater-v2
+        alertname: FlinkJobsLag
+
+    - alert: ingest-router lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) > {{ .Values.ingest_router_threshold_critical}}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for ingest-router job is more than threshold {{ .Values.ingest_router_threshold_critical | int }}
+        summary: ingest-router lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: ingest-router
+        alertname: FlinkJobsLag
+
+### Dp Flink jobs lag alertrules warning
+
+    - alert: telemetry-extractor lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) >  {{ .Values.telemetry_extractor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "telemetry-extractor-taskmanager-prometheus" }) <  {{ .Values.telemetry_extractor_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_threshold_warning | int }}
+        summary: telemetry-extractor lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: telemetry-extractor
+        alertname: FlinkJobsLag
+
+    - alert: pipeline-preprocessor lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) > {{ .Values.pipeline_preprocessor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "pipeline-preprocessor-taskmanager-prometheus" }) < {{ .Values.pipeline_preprocessor_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_threshold_warning | int }}
+        summary: pipeline-preprocessor lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: pipeline-preprocessor
+        alertname: FlinkJobsLag
+   
+    - alert:  de-normalization-secondary lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-secondary-taskmanager-prometheus" }) < {{ .Values.de_normalization_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_threshold_warning  | int }}
+        summary: de-normalization-secondary lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-secondary
+        alertname: FlinkJobsLag
+
+    - alert:  de-normalization-primary lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) > {{ .Values.de_normalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "de-normalization-primary-taskmanager-prometheus" }) < {{ .Values.de_normalization_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for de-normalization-primary job is more than threshold {{ .Values.de_normalization_threshold_warning  | int }}
+        summary: de-normalization-primary lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-primary
+        alertname: FlinkJobsLag
+
+    - alert:  druid-validator lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) > {{ .Values.druid_validator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "druid-validator-taskmanager-prometheus" }) < {{ .Values.druid_validator_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for druid-validator job is more than threshold {{ .Values.druid_validator_threshold_warning | int }}
+        summary: druid-validator lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: druid-validator
+        alertname: FlinkJobsLag
+
+    - alert: assessment-aggregator lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) > {{ .Values.assessment_aggregator_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "assessment-aggregator-taskmanager-prometheus" }) < {{ .Values.assessment_aggregator_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_threshold_warning | int }}
+        summary: assessment-aggregator lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: assessment-aggregator
+        alertname: FlinkJobsLag
+
+    - alert: content-cache-updater lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) > {{ .Values.content_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "content-cache-updater-taskmanager-prometheus" }) < {{ .Values.content_cache_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_threshold_warning | int }}
+        summary: content-cache-updater lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: content-cache-updater
+        alertname: FlinkJobsLag
+
+    - alert: user-cache-updater lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-taskmanager-prometheus" }) < {{ .Values.user_cache_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_threshold_warning | int }}
+        summary: user-cache-updater lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater
+        alertname: FlinkJobsLag
+
+    - alert: summary-denormalization lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) > {{ .Values.summary_denormalization_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "summary-denormalization-taskmanager-prometheus" }) < {{ .Values.summary_denormalization_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_threshold_warning | int }}
+        summary: summary-denormalization lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: summary-denormalization
+        alertname: FlinkJobsLag
+
+    - alert: device-profile-updater lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) > {{ .Values.device_profile_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "device-profile-updater-taskmanager-prometheus" }) < {{ .Values.device_profile_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_threshold_warning | int }}
+        summary: device-profile-updater lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: device-profile-updater
+        alertname: FlinkJobsLag
+
+    - alert: user-cache-updater-v2 lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) > {{ .Values.user_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "user-cache-updater-v2-taskmanager-prometheus" }) < {{ .Values.user_cache_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_updater_threshold_warning | int }}
+        summary: user-cache-updater-v2 lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater-v2
+        alertname: FlinkJobsLag
+
+    - alert: ingest-router lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) > {{ .Values.ingest_router_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "ingest-router-taskmanager-prometheus" }) < {{ .Values.ingest_router_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for ingest-router job is more than threshold {{ .Values.ingest_router_threshold_warning | int }}
+        summary: ingest-router lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: ingest-router
+        alertname: FlinkJobsLag
+
+
+#### KP jobs lag alert rules critical
+    - alert: activity-aggregate-updater lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) > {{ .Values.activity_aggregater_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for activity-aggregate-updater job is more than threshold {{ .Values.activity_aggregater_updater_threshold_critical | int }}
+        summary: activity-aggregate-updater lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: activity-aggregate-updater
+        alertname: FlinkJobsLag
+
+    - alert: relation-cache-updater lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) > {{ .Values.relation_cache_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_threshold_critical | int }}
+        summary: relation-cache-updater lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: relation-cache-updater
+        alertname: FlinkJobsLag
+
+    - alert: post-publish-processor lag critical
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) > {{ .Values.post_publish_processor_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for post-publish-processor job is more than threshold {{ .Values.post_publish_processor_threshold_critical | int }}
+        summary: post-publish-processor lag is critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: post-publish-processor-taskmanager-prometheus
+        alertname: FlinkJobsLag
+
+## KP FLink jobs warning lag alertrules
+
+    - alert: activity-aggregate-updater lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) > {{ .Values.activity_aggregater_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "activity-aggregate-updater-taskmanager-prometheus" }) < {{ .Values.activity_aggregater_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for activity-aggregate-updater job is more than threshold {{ .Values.activity_aggregater_updater_threshold_warning | int }}
+        summary: activity-aggregate-updater lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: activity-aggregate-updater
+        alertname: FlinkJobsLag
+
+    - alert: relation-cache-updater lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) > {{ .Values.relation_cache_updater_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "relation-cache-updater-taskmanager-prometheus" }) < {{ .Values.relation_cache_updater_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_threshold_warning | int }}
+        summary: relation-cache-updater lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: relation-cache-updater
+        alertname: FlinkJobsLag
 
+    - alert: post-publish-processor lag warning
+      expr: sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) > {{ .Values.post_publish_processor_threshold_warning }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "post-publish-processor-taskmanager-prometheus" }) < {{ .Values.post_publish_processor_threshold_critical }}
+      for: 5m
+      labels:
+        severity: warning
+        alerttype: lag
+      annotations:
+        message: The lag for post-publish-processor job is more than threshold {{ .Values.post_publish_processor_threshold_warning | int }}
+        summary: post-publish-processor lag is warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: post-publish-processor
+        alertname: FlinkJobsLag
 #### Checkpoint failure alert rules
     
     - alert: telemetry-extractor checkpoint failure critical
@@ -100,98 +414,208 @@ spec:
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "telemetry-extractor-jobmanager" }) > {{ .Values.telemetry_extractor_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_checkpointfailure_threshold_critical }}
         summary: telemetry-extractor checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: telemetry-extractor
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: pipeline-preprocessor checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "pipeline-preprocessor-jobmanager" }) > {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "pipeline-preprocessor-jobmanager" }) > {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for pipeline-preprocessor job is more than threshold {{ .Values.pipeline_preprocessor_checkpointfailure_threshold_critical }}
         summary: pipeline-preprocessor checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: pipeline-preprocessor
+        alertname: Flink jobs checkpoint commit failure
+
+    - alert: de-normalization-primary checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-primary-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The checkpoint failure count for de-normalization-primary job is more than threshold {{ .Values.de_normalization_checkpointfailure_threshold_critical }}
+        summary: de-normalization-primary checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-primary
+        alertname: Flink jobs checkpoint commit failure
 
-    - alert: de-normalization checkpoint failure critical
-      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }}
+    - alert: de-normalization-secondary checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-secondary-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "de-normalization-jobmanager" }) > {{ .Values.de_normalization_checkpointfailure_threshold_critical }} 
-        summary: de-normalization checkpoint failure critical
+        message: The checkpoint failure count for de-normalization-secondary job is more than threshold {{ .Values.de_normalization_checkpointfailure_threshold_critical }}
+        summary: de-normalization-secondary checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: de-normalization-secondary
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: druid-validator checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "druid-validator-jobmanager" }) > {{ .Values.druid_validator_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "druid-validator-jobmanager" }) > {{ .Values.druid_validator_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for druid-validator job is more than threshold {{ .Values.druid_validator_checkpointfailure_threshold_critical }}
         summary: druid-validator checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: druid-validator
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: assessment-aggregator checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "assessment-aggregator-jobmanager" }) > {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "assessment-aggregator-jobmanager" }) > {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for assessment-aggregator job is more than threshold {{ .Values.assessment_aggregator_checkpointfailure_threshold_critical }}
         summary: assessment-aggregator checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: assessment-aggregator
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: content-cache-updater checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "content-cache-updater-jobmanager" }) > {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "content-cache-updater-jobmanager" }) > {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for content-cache-updater job is more than threshold {{ .Values.content_cache_updater_checkpointfailure_threshold_critical }}
         summary: content-cache-updater checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: content-cache-updater
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: user-cache-updater checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-jobmanager" }) > {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-jobmanager" }) > {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for user-cache-updater job is more than threshold {{ .Values.user_cache_updater_checkpointfailure_threshold_critical }}
         summary: user-cache-updater checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: summary-denormalization checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "summary-denormalization-jobmanager" }) > {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "summary-denormalization-jobmanager" }) > {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for summary-denormalization job is more than threshold {{ .Values.summary_denormalization_checkpointfailure_threshold_critical }}
         summary: summary-denormalization checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: summary-denormalization
+        alertname: Flink jobs checkpoint commit failure
 
     - alert: device-profile-updater checkpoint failure critical
       expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "device-profile-updater-jobmanager" }) > {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }}
       for: 5m
       labels:
         severity: critical
+        alerttype: lag
       annotations:
-        message: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "device_profile_updater-jobmanager" }) > {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }} 
+        message: The checkpoint failure count for device-profile-updater job is more than threshold {{ .Values.device_profile_updater_checkpointfailure_threshold_critical }}
         summary: device-profile-updater checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: device-profile-updater
+        alertname: Flink jobs checkpoint commit failure
+
+    - alert: activity-aggregate-updater checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "activity-aggregate-updater-jobmanager" }) > {{ .Values.activity_aggregate_updater_checkpointfailure_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The checkpoint failure count for device-profile-updater job is more than threshold {{ .Values.activity_aggregate_updater_checkpointfailure_threshold_critical }}
+        summary: activity-aggregate-updater checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: activity-aggregate-updater
+        alertname: Flink jobs checkpoint commit failure
+
+    - alert: relation-cache-updater checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "relation-cache-updater-jobmanager" }) > {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The checkpoint failure count for relation-cache-updater job is more than threshold {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }}
+        summary: relation-cache-updater checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: relation-cache-updater
+        alertname: Flink jobs checkpoint commit failure
+
+    - alert: user-cache-updater-v2 checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "user-cache-updater-v2-jobmanager" }) > {{ .Values.user_cache_v2_updater_checkpointfailure_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The checkpoint failure count for user-cache-updater-v2 job is more than threshold {{ .Values.user_cache_v2_updater_checkpointfailure_threshold_critical }}
+        summary: user-cache-updater-v2 checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: user-cache-updater-v2
+        alertname: Flink jobs checkpoint commit failure
+     
+    - alert: post-publish-processor checkpoint failure critical
+      expr: sum(flink_jobmanager_job_numberOfFailedCheckpoints { job = "post-publish-processor-jobmanager" }) > {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }}
+      for: 5m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The checkpoint failure count for post-publish-processor job is more than threshold {{ .Values.relation_cache_updater_checkpointfailure_threshold_critical }}
+        summary: post-publish-processor checkpoint failure critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: post-publish-processor-jobmanager
+        alertname: Flink jobs checkpoint commit failure
+
 
 ### telemetry extractor failed event percentage alert rule
 
-    - alert: telemetry-extractor failed events percentage fatal
-      expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_fatal }}
+    - alert: telemetry-extractor failed events percentage warning
+      expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_warning }}
       for: 5m
       labels:
-        severity: fatal
+        severity: warning
+        alerttype: lag
       annotations:
-        message: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_fatal }} 
-        summary: telemetry-extractor failed events percentage fatal
+        message: The failed events percentage for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_failed_events_percentage_threshold_warning }} 
+        summary: telemetry-extractor failed events percentage warning
+        lag: {{`({{ humanize $value }})`}}
+        job_id: telemetry-extractor
+        alertname: Flink jobs failed events percentage
 
     - alert: telemetry-extractor failed events percentage critical
       expr: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }}
       for: 5m
       labels:
-        severity: fatal
+        severity: critical
+        alerttype: lag
       annotations:
-        message: sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m])) / (sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_failed_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_success_batch_count[5m]))  + sum(sum_over_time(flink_taskmanager_job_task_operator_TelemetryExtractorJob_duplicate_event_count[5m]))) * 100 > {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }}
+        message: The failed events percentage for telemetry-extractor job is more than threshold {{ .Values.telemetry_extractor_failed_events_percentage_threshold_critical }} 
         summary: telemetry-extractor failed events percentage critical
+        lag: {{`({{ humanize $value }})`}}
+        job_id: telemetry-extractor
+        alertname: Flink jobs failed events percentage
diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml
new file mode 100644
index 0000000000..c4f6e8578c
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/alertrules/templates/flinkjobs_lag.yaml
@@ -0,0 +1,56 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    role: alert-rules
+    app: {{ .Values.prometheus_rule_selector_app }}
+    release: {{ .Values.prometheus_rule_selector_release }}
+  name: {{ .Values.fullnameOverride }}-flinkjobs-lag
+  namespace: {{ .Values.namespace }}
+spec:
+  groups:
+  - name: alertrules.flinkjobs-lag
+    rules:
+    {{ range $key, $threshold := .Values.flink_job_names }}
+    - alert: {{ $key }} lag increase critical
+      expr: ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m ) ) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m) ) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max { job = "{{ $key}}-taskmanager-prometheus" }) >  {{ $threshold }}
+      for: 10m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The lag for {{ $key }} is increased by {{`{{ humanize $value }}`}} percentage
+        summary: {{ $key }} lag is increasing
+        lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })`  | query | first | value | humanize }}'
+        job_id: {{ $key }}
+        alertname: FlinkJobsLag
+    {{- end }}
+
+    {{ range $key, $threshold := .Values.flink_job_names }}
+    - alert: {{ $key }} lag increase percentage critical
+      expr: sum(job:nginx_http_requests_total:rate:sum:5m) - sum(job:nginx_http_requests_total:rate:sum:5m offset {{ $.Values.flink_lag_offset_duration }}m) > {{ $.Values.nginx_tps_threshold }} and ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) > {{ $threshold }}
+      for: 10m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: The tps is increasing and {{ $key }} lag is also increasing. Please check and increase the replica.
+        lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })`  | query | first | value | humanize }}'
+        job_id: {{ $key }}
+        alertname: FlinkJobsPercentageLag
+    {{- end }}
+
+    {{ range $key, $threshold := .Values.flink_job_names }}
+    - alert: {{ $key }} lag increase percentage critical
+      expr: sum(job:nginx_http_requests_total:rate:sum:5m) - sum(job:nginx_http_requests_total:rate:sum:5m offset {{ $.Values.flink_lag_offset_duration }}m) < {{ $.Values.nginx_tps_threshold }} and ((sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) - sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) / sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"} offset {{ $.Values.flink_lag_offset_duration }}m)) * 100 > {{ $.Values.flink_lag_percentage_threshold }} and sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job="{{ $key }}-taskmanager-prometheus"}) > {{ $threshold }}
+      for: 10m
+      labels:
+        severity: critical
+        alerttype: lag
+      annotations:
+        message: There is no increase in tps but {{ $key }} lag is increasing. Please check and restart the job.
+        lag: '{{`{{printf `}}`sum(flink_taskmanager_job_task_operator_KafkaConsumer_records_lag_max{job = "{{ $key }}-taskmanager-prometheus" })`  | query | first | value | humanize }}'
+        job_id: {{ $key }}
+        alertname: FlinkJobsPercentageLag
+    {{- end }}
diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml
index 42ea895d11..c9e7fe89f6 100644
--- a/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml
+++ b/kubernetes/helm_charts/monitoring/alertrules/templates/kubernetesApps.yaml
@@ -14,25 +14,30 @@ spec:
     rules:
     - alert: KubePodCrashLooping
       annotations:
-        message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
-          }}) is restarting {{ printf "%.2f" $value }} times / 5 minutes.`}}
+        message: {{` The pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting. `}}
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping
+        job_id: {{`({{ $labels.container }})`}}
+        alertname: KubePodCrashLooping
       expr: rate(kube_pod_container_status_restarts_total{job="kube-state-metrics",
         namespace=~".*"}[15m]) * 60 * 5 > 0
       for: 15m
       labels:
         severity: critical
+        alerttype: job
     - alert: KubePodNotReady
       annotations:
         message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready`}}
           state for longer than 15 minutes.
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
+        job_id: {{`({{ $labels.pod }})`}}
+        alertname: KubePodNotReady
       expr: sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{job="kube-state-metrics",
         namespace=~".*", phase=~"Pending|Unknown"}) * on(namespace, pod) group_left(owner_kind)
         max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0
       for: 15m
       labels:
         severity: critical
+        alerttype: job
     - alert: KubeDeploymentGenerationMismatch
       annotations:
         message: {{`Deployment generation for {{ $labels.namespace }}/{{ $labels.deployment
@@ -132,11 +137,14 @@ spec:
         message: {{`Pod {{ $labels.namespace }}/{{ $labels.pod }} container {{ $labels.container}}
           has been in waiting state for longer than 1 hour.`}}
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting
+        job_id: {{`({{ $labels.container }})`}}
+        alertname: KubeContainerWaiting
       expr: sum by (namespace, pod, container) (kube_pod_container_status_waiting_reason{job="kube-state-metrics",
         namespace=~".*"}) > 0
       for: 1h
       labels:
         severity: warning
+        alerttype: job
     - alert: KubeDaemonSetNotScheduled
       annotations:
         message: {{`'{{ $value }} Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset
@@ -173,10 +181,13 @@ spec:
       annotations:
         message: {{`Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete.`}}
         runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed
+        job_id: {{`({{ $labels.job_name }})`}}
+        alertname: KubeJobFailed
       expr: kube_job_failed{job="kube-state-metrics", namespace=~".*"}  > 0
       for: 15m
       labels:
-        severity: warning
+        severity: critical
+        alerttype: job
     - alert: KubeHpaReplicasMismatch
       annotations:
         message: {{`HPA {{ $labels.namespace }}/{{ $labels.hpa }} has not matched the desired
@@ -203,3 +214,19 @@ spec:
       for: 15m
       labels:
         severity: warning
+
+## kubepersistentvolumeusagecritical
+    - alert: KubePersistentVolumeUsageCritical
+      annotations:
+        message: The PersistentVolume claimed by {{`{{`}} $labels.persistentvolumeclaim {{`}}`}} in Namespace {{`{{`}} $labels.namespace {{`}}`}} usage is {{`{{`}} printf "%0.2f" $value {{`}}`}}%.
+        metrics: {{`({{ humanize $value }})`}}
+        alertname: KubePersistentVolumeUsageCritical
+      expr: |-
+        100 * kubelet_volume_stats_used_bytes{job="kubelet", namespace="secor"}
+          /
+        kubelet_volume_stats_capacity_bytes{job="kubelet", namespace="secor"}
+          > {{ .Values.pv_usage_threshold_critical }} 
+      for: 1m
+      labels:
+        alerttype: nodemetrics
+        severity: critical
diff --git a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml
index f743a8ea8e..1de7338bda 100644
--- a/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml
+++ b/kubernetes/helm_charts/monitoring/alertrules/templates/promrulesNode.yaml
@@ -13,106 +13,105 @@ spec:
   - name: alertrules.nodes
     rules:
     - alert: high_cpu_usage_on_node_warning
-      expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Warning }} and (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Critical }}
+      expr: (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Warning }} and (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Critical }}
       for: 1m
       labels:
         severity: warning
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}}
         summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}}
+        alertname: HighCpuUsageOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: high_cpu_usage_on_node_critical
-      expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Critical }} and (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) < {{ .Values.node_cpu_usage_percentage_threshold_Fatal }}
+      expr: (avg by (instance) (irate(node_cpu_seconds_total{job="node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Critical }}
       for: 1m
       labels:
         severity: critical
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}}
         summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}}
-    - alert: high_cpu_usage_on_node_fatal
-      expr: (avg by (instance) (irate(node_cpu_seconds_total{job="vm-node-exporter",mode!="idle"}[5m])) * 100) >= {{ .Values.node_cpu_usage_percentage_threshold_Fatal }}
-      for: 1m
-      labels:
-        severity: fatal
-      annotations:
-        message: {{`'{{ $labels.instance }} is using a LOT of CPU. CPU usage is {{ humanize $value}}%.'`}}
-        summary: {{`'HIGH CPU USAGE warning ON {{ $labels.instance }}'`}}
+        alertname: HighCpuUsageOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: high_memory_usage_on_node_warning
       expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Warning }} and (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) < {{ .Values.node_memory_usage_percentage_threshold_Critical }} )
       for: 1m
       labels:
         severity: warning
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}}
         summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}}
+        alertname: HighMemoryUsageOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: high_memory_usage_on_node_critical
-      expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Critical }} and (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) < {{ .Values.node_memory_usage_percentage_threshold_Fatal }} )
+      expr: sum by(nodename) ((((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Critical }})
       for: 1m
       labels:
         severity: critical
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}}
         summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}}
-    - alert: high_memory_usage_on_node_fatal
-      expr: sum by(nodename) (((node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes) * on(instance) group_left(nodename) node_uname_info * 100) >= {{ .Values.node_memory_usage_percentage_threshold_Fatal }}
-      for: 1m
-      labels:
-        severity: fatal
-      annotations:
-        message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) is using a LOT of MEMORY. MEMORY usage is over {{ humanize $value}}.'`}}
-        summary: {{`'HIGH MEMORY USAGE warning TASK ON {{ $labels.nodename }}'`}}
+        alertname: HighMemoryUsageOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: high_load_on_node_warning
-      expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Warning }} and (node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Critical }})
+      expr: sum by(instance) ((node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Warning }} and (node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Critical }})
       for: 5m
       labels:
         severity: warning
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}}
         summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}}
+        alertname: HighLoadOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: high_load_on_node_critical
-      expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Critical }} and (node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) < {{ .Values.node_load_avg_threshold_Fatal }})
+      expr: sum by(instance) ((node_load1{job="node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Critical }})
       for: 5m
       labels:
         severity: critical
+        alerttype: nodemetrics
       annotations:
         message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}}
         summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}}
-    - alert: high_load_on_node_fatal
-      expr: sum by(instance) ((node_load1{job="vm-node-exporter"} / count by (cluster, job, instance)(count by(cluster, job, instance, cpu)(node_cpu_seconds_total{job="vm-node-exporter"})) * 100) >= {{ .Values.node_load_avg_threshold_Fatal }})
-      for: 5m
-      labels:
-        severity: fatal
-      annotations:
-        message: {{`'{{ $labels.nodename }} ({{ $labels.host }}) has a high load average. Load average is {{ $value }}%.'`}}
-        summary: {{`'HIGH LOAD AVERAGE warning ON {{ $labels.nodename }}'`}}
+        alertname: HighLoadOnK8sNode
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: node_exporter_down_critical
       expr: up == 0
       for: 1m
       labels:
-        severity: critical
+        severity: warning
       annotations:
         message: {{`The node exporter '{{ $labels.job }}' is down.`}}
         summary: {{`'NODE EXPORTER SERVICE critical: NODE ''{{ $labels.host }}'''`}}
+
     - alert: node_running_out_of_disk_space_warning
       expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Warning }} and ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) < {{ .Values.node_disk_usage_percentage_threshold_Critical }} )
       for: 1m
       labels:
         severity: warning
+        alerttype: nodemetrics
       annotations:
         message: {{`'Disk usage is {{ humanize $value }}%'`}}
         summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}}
+        alertname: K8sNodeRunningOutOfDiskSpace
+        metrics: {{`({{ humanize $value }})`}}%
+
     - alert: node_running_out_of_disk_space_warning
-      expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Critical }} and ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) < {{ .Values.node_disk_usage_percentage_threshold_Fatal }} )
+      expr: sum by(nodename) (((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Critical }})
       for: 1m
       labels:
         severity: critical
       annotations:
         message: {{`'Disk usage is {{ humanize $value }}%'`}}
         summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}}
-    - alert: node_running_out_of_disk_space_fatal
-      expr: sum by(nodename) ((node_filesystem_size_bytes{mountpoint="/"} - node_filesystem_free_bytes{mountpoint="/"}) * 100 / node_filesystem_size_bytes{mountpoint="/"} * on(instance) group_left(nodename) node_uname_info) >= {{ .Values.node_disk_usage_percentage_threshold_Fatal }}
-      for: 1m
-      labels:
-        severity: fatal
-      annotations:
-        message: {{`'Disk usage is {{ humanize $value }}%'`}}
-        summary: {{`'LOW DISK SPACE WARING: NODE {{ $labels.nodename }}'`}}
+        alertname: K8sNodeRunningOutOfDiskSpace 
+        metrics: {{`({{ humanize $value }})`}}%
diff --git a/kubernetes/helm_charts/monitoring/alertrules/values.yaml b/kubernetes/helm_charts/monitoring/alertrules/values.yaml
index 990ed4e8ef..d6c247afc2 100644
--- a/kubernetes/helm_charts/monitoring/alertrules/values.yaml
+++ b/kubernetes/helm_charts/monitoring/alertrules/values.yaml
@@ -5,15 +5,15 @@ fullnameOverride: sunbird-monitoring
 namespace: monitoring
 
 # Node Exporter vars
-node_cpu_usage_percentage_threshold_Warning: 75
-node_cpu_usage_percentage_threshold_Critical: 85
+node_cpu_usage_percentage_threshold_Warning: 80
+node_cpu_usage_percentage_threshold_Critical: 90
 node_cpu_usage_percentage_threshold_Fatal: 95
-node_memory_usage_percentage_threshold_Critical: 85
-node_memory_usage_percentage_threshold_Warning: 75
+node_memory_usage_percentage_threshold_Critical: 90
+node_memory_usage_percentage_threshold_Warning: 80
 node_memory_usage_percentage_threshold_Fatal: 95
-node_load_avg_threshold_Critical: 95
-node_load_avg_threshold_Warning: 85
-node_load_avg_threshold_Fatal: 120
-node_disk_usage_percentage_threshold_Warning: 75
-node_disk_usage_percentage_threshold_Critical: 85
+node_load_avg_threshold_Critical: 90
+node_load_avg_threshold_Warning: 80
+node_load_avg_threshold_Fatal: 95
+node_disk_usage_percentage_threshold_Warning: 80
+node_disk_usage_percentage_threshold_Critical: 90
 node_disk_usage_percentage_threshold_Fatal: 95
diff --git a/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter b/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter
new file mode 120000
index 0000000000..8ed35ad576
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/ingestion-kafka-exporter
@@ -0,0 +1 @@
+processing-kafka-exporter
\ No newline at end of file
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore
new file mode 100644
index 0000000000..50af031725
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml
new file mode 100644
index 0000000000..3a5191414d
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/Chart.yaml
@@ -0,0 +1,9 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes
+name: processing-kafka-exporter
+version: 1.0.0
+home: https://github.com/abhishekjiitr/kafka-exporter-helm
+maintainers:
+  - name: abhishekjiitr
+    email: abhi2254015@gmail.com
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt
new file mode 100644
index 0000000000..d9eb9e809b
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/NOTES.txt
@@ -0,0 +1,5 @@
+1.To see the metrics
+{{- if contains "ClusterIP" .Values.service.type }}
+  kubectl port-forward svc/{{ include "kafka-exporter.fullname" . }} {{ .Values.service.port }}
+  echo "Visit http://127.0.0.1:{{ .Values.service.port }} to use your application"
+{{- end }}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl
new file mode 100644
index 0000000000..bc51bbfcd5
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/_helpers.tpl
@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "kafka-exporter.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "kafka-exporter.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "kafka-exporter.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml
new file mode 100644
index 0000000000..e107a6a78d
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/alertRules.yaml
@@ -0,0 +1,62 @@
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: {{ include "kafka-exporter.fullname" . }}
+  {{- if .Values.prometheus.serviceMonitor.namespace }}
+  namespace: {{ .Values.prometheus.serviceMonitor.namespace }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    {{- if .Values.prometheus.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.prometheus.serviceMonitor.additionalLabels | indent 4 -}}
+    {{- end }}
+spec:
+  groups:
+  - name: alertrules.kafkaExporter
+    rules:
+    - record: isr_partition_difference
+      expr: kafka_topic_partition_in_sync_replica{topic!="__consumer_offsets",topic!="__samza_*"} - kafka_topic_partition_replicas{topic!="__consumer_offsets",topic!="__samza_*"}
+
+    - alert: ISR_not_equal_to_partition
+      annotations:
+        message: {{`"ISR and replication difference for $labels.topic is > 0 current value: {{ humanize $value }}"`}}
+        summary: {{`"ISR and Replica Mismatch for {{$labels.topic}}"`}}
+      expr: isr_partition_difference != 0
+      for: 5m
+      labels:
+        severity: critical
+        module: dp
+
+    - alert: Kafka_partition_leader_change_rapidly
+      annotations:
+        message:  {{`"Kafaka Partition leader changing rapidly for {{ $labels.topic }}"`}}
+        summary: {{`"Kafaka Partition leader changing rapidly for {{ $labels.topic }}"`}}
+      expr: rate(kafka_topic_partition_leader{topic!="__consumer_offsets",topic!="__samza_*"}[5m]) > 0
+      for: 5m
+      labels:
+        severity: critical
+        module: dp
+
+    - alert: kafka_broker_unavailable_processing_cluster
+      annotations:
+        summary: {{`"Kafka brokers unavailable processing cluster"`}}
+        message: "There are only {{`{{humanize $value}}`}} kafka brokers available; Expected count: {{ len .Values.kafkaExporter.kafka.servers }}"
+      expr: kafka_brokers{job="processing-kafka-exporter"} < {{ len .Values.kafkaExporter.kafka.servers }}
+      for: 5m
+      labels:
+        severity: critical
+        module: dp
+        
+    - alert: kafka_broker_unavailable_ingestion_cluster
+      annotations:
+        summary: {{`"Kafka brokers unavailable ingestion cluster "`}}
+        message: "There are only {{`{{humanize $value}}`}} kafka brokers available; Expected count: {{ len .Values.ingestion_cluster_kafka_server_count }}"
+      expr: kafka_brokers{job="ingestion-kafka-exporter"} < {{ len .Values.ingestion_cluster_kafka_server_count }}
+      for: 5m
+      labels:
+        severity: critical
+        module: dp
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml
new file mode 100644
index 0000000000..d79b57b019
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/deployment.yaml
@@ -0,0 +1,91 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "kafka-exporter.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  replicas: {{ .Values.replicaCount }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+    spec:
+      containers:
+        - name: {{ .Chart.Name }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          args:
+            {{- if .Values.kafkaExporter}}
+            {{- range .Values.kafkaExporter.kafka.servers }}
+            - "--kafka.server={{ . }}"
+            {{- end }}
+            {{- range .Values.kafkaExporter.zookeeper.servers }}
+            - "--zookeeper.server={{ . }}"
+            {{- end }}
+            {{- range .Values.kafkaExporter.additionalFlags }}
+            - "{{ . }}"
+            {{- end }}
+            {{- if .Values.kafkaExporter.kafka.version }}
+            - --kafka.version={{ .Values.kafkaExporter.kafka.version }}
+            {{- end }}
+            {{- end}}
+            {{- if .Values.kafkaExporter.sasl.enabled }}
+            - --sasl.enabled
+            {{- if not .Values.kafkaExporter.sasl.handshake }}
+            - --sasl.handshake=false
+            {{- end }}
+            - --sasl.username={{ .Values.kafkaExporter.sasl.username }}
+            - --sasl.password={{ .Values.kafkaExporter.sasl.password }}
+            {{- end }}
+            {{- if .Values.kafkaExporter.tls.enabled}}
+            - --tls.enabled
+            - --tls.ca-file=/etc/tls-certs/ca-file
+            - --tls.cert-file=/etc/tls-certs/cert-file
+            - --tls.key-file=/etc/tls-certs/key-file
+            {{- end }}
+            {{- if .Values.kafkaExporter.log }}
+            - --log.level={{ .Values.kafkaExporter.log.level }}
+            {{- if .Values.kafkaExporter.log.enableSarama }}
+            - --log.enable-sarama
+            {{- end }}
+            {{- end }}
+          ports:
+            - name: metrics
+              containerPort: 9308
+              protocol: TCP
+          {{- if .Values.kafkaExporter.tls.enabled }}
+          volumeMounts:
+          - name: tls-certs
+            mountPath: "/etc/tls-certs/"
+            readOnly: true
+          {{- end }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+      {{- with .Values.nodeSelector }}
+
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+    {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+    {{- end }}
+    {{- if .Values.kafkaExporter.tls.enabled }}
+      volumes:
+      - name: tls-certs
+        secret:
+          secretName: {{ include "kafka-exporter.fullname" . }}
+    {{- end }}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml
new file mode 100644
index 0000000000..82f567f38d
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/secret.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.kafkaExporter.tls.enabled }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "kafka-exporter.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  ca-file: {{ .Values.kafkaExporter.tls.caFile | b64enc }}
+  cert-file: {{ .Values.kafkaExporter.tls.certFile | b64enc }}
+  key-file: {{ .Values.kafkaExporter.tls.keyFile | b64enc }}
+{{- end }}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml
new file mode 100644
index 0000000000..049041fb03
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/service.yaml
@@ -0,0 +1,19 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "kafka-exporter.fullname" . }}
+  labels:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+spec:
+  type: {{ .Values.service.type }}
+  ports:
+    - port: {{ .Values.service.port }}
+      targetPort: metrics
+      protocol: TCP
+      name: metrics
+  selector:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml
new file mode 100644
index 0000000000..395d617dd3
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/templates/servicemonitor.yaml
@@ -0,0 +1,33 @@
+{{- if .Values.prometheus.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ include "kafka-exporter.fullname" . }}
+  {{- if .Values.prometheus.serviceMonitor.namespace }}
+  namespace: {{ .Values.prometheus.serviceMonitor.namespace }}
+  {{- end }}
+  labels:
+    app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+    helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    {{- if .Values.prometheus.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.prometheus.serviceMonitor.additionalLabels | indent 4 -}}
+    {{- end }}
+spec:
+  jobLabel: jobLabel
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ include "kafka-exporter.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      helm.sh/chart: {{ include "kafka-exporter.chart" . }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  endpoints:
+  - port: metrics
+    interval: {{ .Values.prometheus.serviceMonitor.interval }}
+    {{- if .Values.prometheus.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.prometheus.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml
new file mode 100644
index 0000000000..2431eafa2e
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/processing-kafka-exporter/values.yaml
@@ -0,0 +1,70 @@
+# Default values for kafka-exporter.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+replicaCount: 1
+ingestion_cluster_kafka_server_count: ["{{ ingestion_cluster_kafka | join('","') }}"]
+
+image:
+  repository: danielqsj/kafka-exporter
+  tag: latest
+  pullPolicy: IfNotPresent
+
+nameOverride: ""
+fullnameOverride: ""
+
+service:
+  type: ClusterIP
+  port: 9308
+
+kafkaExporter:
+  kafka:
+    servers: []
+  zookeeper:
+    servers: []
+  additionalFlags: []
+    # - --use.consumelag.zookeeper
+
+  sasl:
+    enabled: false
+    handshake: true
+    username: ""
+    password: ""
+
+  tls:
+    enabled: false
+    insecure-skip-tls-verify: false
+    caFile: ""
+    certFile: ""
+    keyFile: ""
+
+  log:
+    level: info
+    enableSarama: false
+
+prometheus:
+  serviceMonitor:
+    enabled: true
+    namespace: monitoring
+    interval: "30s"
+    additionalLabels:
+      app: kafka-exporter
+
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore b/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore
new file mode 100755
index 0000000000..f0c1319444
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/.helmignore
@@ -0,0 +1,21 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml
new file mode 100755
index 0000000000..96efec0851
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/Chart.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+appVersion: v0.7.0
+deprecated: true
+description: DEPRECATED A Helm chart for k8s prometheus adapter
+home: https://github.com/DirectXMan12/k8s-prometheus-adapter
+keywords:
+- hpa
+- metrics
+- prometheus
+- adapter
+name: prometheus-adapter
+sources:
+- https://github.com/kubernetes/charts
+- https://github.com/DirectXMan12/k8s-prometheus-adapter
+version: 2.5.1
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS b/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS
new file mode 100755
index 0000000000..0c15fcf344
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/OWNERS
@@ -0,0 +1,8 @@
+approvers:
+  - mattiasgees
+  - steven-sheehy
+  - hectorj2f
+reviewers:
+  - mattiasgees
+  - steven-sheehy
+  - hectorj2f
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md b/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md
new file mode 100755
index 0000000000..182e633d5a
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/README.md
@@ -0,0 +1,162 @@
+# Prometheus Adapter
+
+DEPRECATED and moved to <https://github.com/prometheus-community/helm-charts>
+
+Installs the [Prometheus Adapter](https://github.com/DirectXMan12/k8s-prometheus-adapter) for the Custom Metrics API. Custom metrics are used in Kubernetes by [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) to scale workloads based upon your own metric pulled from an external metrics provider like Prometheus. This chart complements the [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server) chart that provides resource only metrics.
+
+## Prerequisites
+
+Kubernetes 1.14+
+
+## Installing the Chart
+
+To install the chart with the release name `my-release`:
+
+```console
+$ helm install --name my-release stable/prometheus-adapter
+```
+
+This command deploys the prometheus adapter with the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation.
+
+## Using the Chart
+
+To use the chart, ensure the `prometheus.url` and `prometheus.port` are configured with the correct Prometheus service endpoint. If Prometheus is exposed under HTTPS the host's CA Bundle must be exposed to the container using `extraVolumes` and `extraVolumeMounts`.
+
+Additionally, the chart comes with a set of default rules out of the box but they may pull in too many metrics or not map them correctly for your needs. Therefore, it is recommended to populate `rules.custom` with a list of rules (see the [config document](https://github.com/DirectXMan12/k8s-prometheus-adapter/blob/master/docs/config.md) for the proper format).
+
+Finally, to configure your Horizontal Pod Autoscaler to use the custom metric, see the custom metrics section of the [HPA walkthrough](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-multiple-metrics-and-custom-metrics).
+
+The Prometheus Adapter can serve three different [metrics APIs](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/#support-for-metrics-apis):
+
+### Custom Metrics
+
+Enabling this option will cause custom metrics to be served at `/apis/custom.metrics.k8s.io/v1beta1`. Enabled by default when `rules.default` is true, but can be customized by populating `rules.custom`:
+
+```
+rules:
+  custom:
+  - seriesQuery: '{__name__=~"^some_metric_count$"}'
+    resources:
+      template: <<.Resource>>
+    name:
+      matches: ""
+      as: "my_custom_metric"
+    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+```
+
+### External Metrics
+
+Enabling this option will cause external metrics to be served at `/apis/external.metrics.k8s.io/v1beta1`. Can be enabled by populating `rules.external`:
+
+```
+rules:
+  external:
+  - seriesQuery: '{__name__=~"^some_metric_count$"}'
+    resources:
+      template: <<.Resource>>
+    name:
+      matches: ""
+      as: "my_external_metric"
+    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+```
+
+### Resource Metrics
+
+Enabling this option will cause resource metrics to be served at `/apis/metrics.k8s.io/v1beta1`. Resource metrics will allow pod CPU and Memory metrics to be used in [Horizontal Pod Autoscalers](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) as well as the `kubectl top` command. Can be enabled by populating `rules.resource`:
+
+```
+rules:
+  resource:
+    cpu:
+      containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>)
+      nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>)
+      resources:
+        overrides:
+          instance:
+            resource: node
+          namespace:
+            resource: namespace
+          pod:
+            resource: pod
+      containerLabel: container
+    memory:
+      containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+      nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+      resources:
+        overrides:
+          instance:
+            resource: node
+          namespace:
+            resource: namespace
+          pod:
+            resource: pod
+      containerLabel: container
+    window: 3m
+```
+
+**NOTE:** Setting a value for `rules.resource` will also deploy the resource metrics API service, providing the same functionality as [metrics-server](https://github.com/helm/charts/tree/master/stable/metrics-server). As such it is not possible to deploy them both in the same cluster.
+
+## Uninstalling the Chart
+
+To uninstall/delete the `my-release` deployment:
+
+```console
+$ helm delete my-release
+```
+
+The command removes all the Kubernetes components associated with the chart and deletes the release.
+
+## Configuration
+
+The following table lists the configurable parameters of the Prometheus Adapter chart and their default values.
+
+| Parameter                       | Description                                                                     | Default                                     |
+| ------------------------------- | ------------------------------------------------------------------------------- | --------------------------------------------|
+| `affinity`                      | Node affinity                                                                   | `{}`                                        |
+| `image.repository`              | Image repository                                                                | `directxman12/k8s-prometheus-adapter-amd64` |
+| `image.tag`                     | Image tag                                                                       | `v0.7.0`                                    |
+| `image.pullPolicy`              | Image pull policy                                                               | `IfNotPresent`                              |
+| `image.pullSecrets`             | Image pull secrets                                                              | `{}`                                        |
+| `logLevel`                      | Log level                                                                       | `4`                                         |
+| `listenPort`                    | Port that application would listen on in the container                          | `6443`                                      |
+| `metricsRelistInterval`         | Interval at which to re-list the set of all available metrics from Prometheus   | `1m`                                        |
+| `nodeSelector`                  | Node labels for pod assignment                                                  | `{}`                                        |
+| `podLabels`                     | Labels to add to the pod                                                        | `{}`                                        |
+| `podAnnotations`                | Annotations to add to the pod                                                   | `{}`                                        |
+| `priorityClassName`             | Pod priority                                                                    | ``                                          |
+| `prometheus.url`                | Url of where we can find the Prometheus service                                 | `http://prometheus.default.svc`             |
+| `prometheus.port`               | Port of where we can find the Prometheus service, zero to omit this option      | `9090`                                      |
+| `prometheus.path`               | Custom path to append to the prometheus URL                                     | ``                                          |
+| `rbac.create`                   | If true, create & use RBAC resources                                            | `true`                                      |
+| `resources`                     | CPU/Memory resource requests/limits                                             | `{}`                                        |
+| `rules.default`                 | If `true`, enable a set of default rules in the configmap                       | `true`                                      |
+| `rules.custom`                  | A list of custom configmap rules                                                | `[]`                                        |
+| `rules.existing`                | The name of an existing configMap with rules. Overrides default, custom and external. | ``                                    |
+| `rules.external`                | A list of custom rules for external metrics API                                 | `[]`                                        |
+| `rules.resource`                | `resourceRules` to set in configmap rules                                       | `{}`                                        |
+| `service.annotations`           | Annotations to add to the service                                               | `{}`                                        |
+| `service.port`                  | Service port to expose                                                          | `443`                                       |
+| `service.type`                  | Type of service to create                                                       | `ClusterIP`                                 |
+| `serviceAccount.create`         | If true, create & use Serviceaccount                                            | `true`                                      |
+| `serviceAccount.name`           | If not set and create is true, a name is generated using the fullname template  | ``                                          |
+| `tls.enable`                    | If true, use the provided certificates. If false, generate self-signed certs    | `false`                                     |
+| `tls.ca`                        | Public CA file that signed the APIService (ignored if tls.enable=false)         | ``                                          |
+| `tls.key`                       | Private key of the APIService (ignored if tls.enable=false)                     | ``                                          |
+| `tls.certificate`               | Public key of the APIService (ignored if tls.enable=false)                      | ``                                          |
+| `extraVolumeMounts`             | Any extra volumes mounts                                                        | `[]`                                        |
+| `extraVolumes`                  | Any extra volumes                                                               | `[]`                                        |
+| `tolerations`                   | List of node taints to tolerate                                                 | `[]`                                        |
+
+Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example,
+
+```console
+$ helm install --name my-release \
+  --set logLevel=1 \
+ stable/prometheus-adapter
+```
+
+Alternatively, a YAML file that specifies the values for the above parameters can be provided while installing the chart. For example,
+
+```console
+$ helm install --name my-release -f values.yaml stable/prometheus-adapter
+```
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/default-values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/default-values.yaml
new file mode 100755
index 0000000000..e69de29bb2
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml
new file mode 100755
index 0000000000..2dafb56211
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/ci/external-rules-values.yaml
@@ -0,0 +1,9 @@
+rules:
+  external:
+  - seriesQuery: '{__name__=~"^some_metric_count$"}'
+    resources:
+      template: <<.Resource>>
+    name:
+      matches: ""
+      as: "my_custom_metric"
+    metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt
new file mode 100755
index 0000000000..329148be71
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/NOTES.txt
@@ -0,0 +1,11 @@
+DEPRECATED and moved to <https://github.com/prometheus-community/helm-charts>
+
+{{ template "k8s-prometheus-adapter.fullname" .  }} has been deployed.
+In a few minutes you should be able to list metrics using the following command(s):
+{{ if .Values.rules.resource }}
+  kubectl get --raw /apis/metrics.k8s.io/v1beta1
+{{- end }}
+  kubectl get --raw /apis/custom.metrics.k8s.io/v1beta1
+{{ if .Values.rules.external }}
+  kubectl get --raw /apis/external.metrics.k8s.io/v1beta1
+{{- end }}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl
new file mode 100755
index 0000000000..5d0b993c52
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/_helpers.tpl
@@ -0,0 +1,43 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "k8s-prometheus-adapter.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "k8s-prometheus-adapter.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "k8s-prometheus-adapter.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "k8s-prometheus-adapter.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "k8s-prometheus-adapter.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
new file mode 100755
index 0000000000..44761c4569
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-delegator-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}:system:auth-delegator
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:auth-delegator
+subjects:
+- kind: ServiceAccount
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml
new file mode 100755
index 0000000000..79c0af39f6
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-auth-reader-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-auth-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: extension-apiserver-authentication-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml
new file mode 100755
index 0000000000..2f828f4a6d
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-deployment.yaml
@@ -0,0 +1,120 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.fullname" . }}
+spec:
+  replicas: {{ .Values.replicas }}
+  selector:
+    matchLabels:
+      app: {{ template "k8s-prometheus-adapter.name" . }}
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: {{ template "k8s-prometheus-adapter.name" . }}
+        chart: {{ template "k8s-prometheus-adapter.chart" . }}
+        release: {{ .Release.Name }}
+        heritage: {{ .Release.Service }}
+        {{- with .Values.podLabels }}
+        {{- toYaml . | trim | nindent 8 }}
+        {{- end }}
+      name: {{ template "k8s-prometheus-adapter.name" . }}
+      annotations:
+        checksum/config: {{ include (print $.Template.BasePath "/custom-metrics-configmap.yaml") . | sha256sum }}
+        {{- with .Values.podAnnotations }}
+        {{- toYaml . | trim | nindent 8 }}
+        {{- end }}
+    spec:
+      serviceAccountName: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+      {{- if .Values.hostNetwork.enabled }}
+      hostNetwork: true
+      {{- end }}
+      containers:
+      - name: {{ .Chart.Name }}
+        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+        imagePullPolicy: {{ .Values.image.pullPolicy }}
+        args:
+        - /adapter
+        - --secure-port={{ .Values.listenPort }}
+        {{- if .Values.tls.enable }}
+        - --tls-cert-file=/var/run/serving-cert/tls.crt
+        - --tls-private-key-file=/var/run/serving-cert/tls.key
+        {{- end }}
+        - --cert-dir=/tmp/cert
+        - --logtostderr=true
+        - --prometheus-url={{ .Values.prometheus.url }}{{ if .Values.prometheus.port }}:{{ .Values.prometheus.port }}{{end}}{{ .Values.prometheus.path }}
+        - --metrics-relist-interval={{ .Values.metricsRelistInterval }}
+        - --v={{ .Values.logLevel }}
+        - --config=/etc/adapter/config.yaml
+        ports:
+        - containerPort: {{ .Values.listenPort }}
+          name: https
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 30
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: https
+            scheme: HTTPS
+          initialDelaySeconds: 30
+        {{- if .Values.resources }}
+        resources:
+          {{- toYaml .Values.resources | nindent 10 }}
+        {{- end }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["all"]
+          readOnlyRootFilesystem: true
+          runAsNonRoot: true
+          runAsUser: 10001
+        volumeMounts:
+        {{- if .Values.extraVolumeMounts }}
+        {{ toYaml .Values.extraVolumeMounts | trim | nindent 8 }}
+        {{ end }}
+        - mountPath: /etc/adapter/
+          name: config
+          readOnly: true
+        - mountPath: /tmp
+          name: tmp
+        {{- if .Values.tls.enable }}
+        - mountPath: /var/run/serving-cert
+          name: volume-serving-cert
+          readOnly: true
+        {{- end }}
+      nodeSelector:
+        {{- toYaml .Values.nodeSelector | nindent 8 }}
+      affinity:
+        {{- toYaml .Values.affinity | nindent 8 }}
+      priorityClassName: {{ .Values.priorityClassName }}
+      tolerations:
+        {{- toYaml .Values.tolerations | nindent 8 }}
+      {{- if .Values.image.pullSecrets }}
+      imagePullSecrets:
+        {{- range .Values.image.pullSecrets }}
+        - name: {{ . }}
+        {{- end }}
+      {{- end }}
+      volumes:
+      {{- if .Values.extraVolumes  }}
+      {{ toYaml .Values.extraVolumes | trim | nindent 6 }}
+      {{ end }}
+      - name: config
+        configMap:
+          name: {{ .Values.rules.existing | default (include "k8s-prometheus-adapter.fullname" . ) }}
+      - name: tmp
+        emptyDir: {}
+      {{- if .Values.tls.enable }}
+      - name: volume-serving-cert
+        secret:
+          secretName: {{ template "k8s-prometheus-adapter.fullname" . }}
+      {{- end }}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
new file mode 100755
index 0000000000..ec7e5e4762
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-resource-reader-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader
+subjects:
+- kind: ServiceAccount
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml
new file mode 100755
index 0000000000..d73deacb78
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service-account.yaml
@@ -0,0 +1,11 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml
new file mode 100755
index 0000000000..83dd70a91b
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiserver-service.yaml
@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+{{ toYaml .Values.service.annotations | indent 4 }}
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.fullname" . }}
+spec:
+  ports:
+  - port: {{ .Values.service.port }}
+    protocol: TCP
+    targetPort: https
+  selector:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    release: {{ .Release.Name }}
+  type: {{ .Values.service.type }}
+
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml
new file mode 100755
index 0000000000..1eb380d70f
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-apiservice.yaml
@@ -0,0 +1,23 @@
+{{- if or .Values.rules.default .Values.rules.custom }}
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: v1beta1.custom.metrics.k8s.io
+spec:
+  service:
+    name: {{ template "k8s-prometheus-adapter.fullname" . }}
+    namespace: {{ .Release.Namespace | quote }}
+  {{ if .Values.tls.enable -}}
+  caBundle: {{ b64enc .Values.tls.ca }}
+  {{- end }}
+  group: custom.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }}
+  groupPriorityMinimum: 100
+  versionPriority: 100
+{{- end }}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml
new file mode 100755
index 0000000000..33daf7113b
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-cluster-role.yaml
@@ -0,0 +1,16 @@
+{{- if and .Values.rbac.create (or .Values.rules.default .Values.rules.custom) -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources
+rules:
+- apiGroups:
+  - custom.metrics.k8s.io
+  resources: ["*"]
+  verbs: ["*"]
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml
new file mode 100755
index 0000000000..a191e8add3
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-configmap.yaml
@@ -0,0 +1,95 @@
+{{- if not .Values.rules.existing -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "k8s-prometheus-adapter.fullname" . }}
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  config.yaml: |
+{{- if or .Values.rules.default .Values.rules.custom }}
+    rules:
+{{- if .Values.rules.default }}
+    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
+      seriesFilters: []
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod:
+            resource: pod
+      name:
+        matches: ^container_(.*)_seconds_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m]))
+        by (<<.GroupBy>>)
+    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
+      seriesFilters:
+      - isNot: ^container_.*_seconds_total$
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod:
+            resource: pod
+      name:
+        matches: ^container_(.*)_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container!="POD"}[5m]))
+        by (<<.GroupBy>>)
+    - seriesQuery: '{__name__=~"^container_.*",container!="POD",namespace!="",pod!=""}'
+      seriesFilters:
+      - isNot: ^container_.*_total$
+      resources:
+        overrides:
+          namespace:
+            resource: namespace
+          pod:
+            resource: pod
+      name:
+        matches: ^container_(.*)$
+        as: ""
+      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container!="POD"}) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters:
+      - isNot: .*_total$
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ""
+        as: ""
+      metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters:
+      - isNot: .*_seconds_total
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ^(.*)_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)
+    - seriesQuery: '{namespace!="",__name__!~"^container_.*"}'
+      seriesFilters: []
+      resources:
+        template: <<.Resource>>
+      name:
+        matches: ^(.*)_seconds_total$
+        as: ""
+      metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>)
+{{- end -}}
+{{- if .Values.rules.custom }}
+{{ toYaml .Values.rules.custom | indent 4 }}
+{{- end -}}
+{{- end -}}
+{{- if .Values.rules.external }}
+    externalRules:
+{{ toYaml .Values.rules.external | indent 4 }}
+{{- end -}}
+{{- if .Values.rules.resource }}
+    resourceRules:
+{{ toYaml .Values.rules.resource | indent 6 }}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml
new file mode 100755
index 0000000000..319460a332
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/custom-metrics-resource-reader-cluster-role.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-resource-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - namespaces
+  - pods
+  - services
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml
new file mode 100755
index 0000000000..560b2ffcd8
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-apiservice.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.rules.external }}
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: v1beta1.external.metrics.k8s.io
+spec:
+  service:
+    name: {{ template "k8s-prometheus-adapter.fullname" . }}
+    namespace: {{ .Release.Namespace | quote }}
+  {{ if .Values.tls.enable -}}
+  caBundle: {{ b64enc .Values.tls.ca }}
+  {{- end }}
+  group: external.metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }}
+  groupPriorityMinimum: 100
+  versionPriority: 100
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml
new file mode 100755
index 0000000000..4adbd65372
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/external-metrics-cluster-role.yaml
@@ -0,0 +1,20 @@
+{{- if and .Values.rbac.create .Values.rules.external -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics
+rules:
+- apiGroups:
+  - "external.metrics.k8s.io"
+  resources:
+  - "*"
+  verbs:
+  - list
+  - get
+  - watch
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml
new file mode 100755
index 0000000000..641b9d3a44
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-custom-metrics-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if .Values.rbac.create -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "k8s-prometheus-adapter.name" . }}-server-resources
+subjects:
+- kind: ServiceAccount
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml
new file mode 100755
index 0000000000..0776029af2
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/hpa-external-metrics-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.rbac.create .Values.rules.external -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-external-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "k8s-prometheus-adapter.name" . }}-external-metrics
+subjects:
+- kind: ServiceAccount
+  name: horizontal-pod-autoscaler
+  namespace: kube-system
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml
new file mode 100755
index 0000000000..f1a619912f
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-apiservice.yaml
@@ -0,0 +1,23 @@
+{{- if .Values.rules.resource}}
+apiVersion: apiregistration.k8s.io/v1beta1
+kind: APIService
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: v1beta1.metrics.k8s.io
+spec:
+  service:
+    name: {{ template "k8s-prometheus-adapter.fullname" . }}
+    namespace: {{ .Release.Namespace | quote }}
+  {{ if .Values.tls.enable -}}
+  caBundle: {{ b64enc .Values.tls.ca }}
+  {{- end }}
+  group: metrics.k8s.io
+  version: v1beta1
+  insecureSkipTLSVerify: {{ if .Values.tls.enable }}false{{ else }}true{{ end }}
+  groupPriorityMinimum: 100
+  versionPriority: 100
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml
new file mode 100755
index 0000000000..0534af11e6
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role-binding.yaml
@@ -0,0 +1,19 @@
+{{- if and .Values.rbac.create .Values.rules.resource -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-hpa-controller-metrics
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ template "k8s-prometheus-adapter.name" . }}-metrics
+subjects:
+- kind: ServiceAccount
+  name: {{ template "k8s-prometheus-adapter.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml
new file mode 100755
index 0000000000..01a307d69b
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/resource-metrics-cluster-role.yaml
@@ -0,0 +1,22 @@
+{{- if and .Values.rbac.create .Values.rules.resource -}}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.name" . }}-metrics
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - nodes
+  - nodes/stats
+  verbs:
+  - get
+  - list
+  - watch
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml
new file mode 100755
index 0000000000..38e7cb624f
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/templates/secret.yaml
@@ -0,0 +1,15 @@
+{{- if .Values.tls.enable -}}
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app: {{ template "k8s-prometheus-adapter.name" . }}
+    chart: {{ template "k8s-prometheus-adapter.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+  name: {{ template "k8s-prometheus-adapter.fullname" . }}
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ b64enc .Values.tls.certificate }}
+  tls.key: {{ b64enc .Values.tls.key }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml b/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml
new file mode 100755
index 0000000000..4cc5e40692
--- /dev/null
+++ b/kubernetes/helm_charts/monitoring/prometheus-adapter/values.yaml
@@ -0,0 +1,141 @@
+# Default values for k8s-prometheus-adapter..
+affinity: {}
+
+image:
+  repository: directxman12/k8s-prometheus-adapter-amd64
+  tag: v0.7.0
+  pullPolicy: IfNotPresent
+
+logLevel: 4
+
+metricsRelistInterval: 1m
+
+listenPort: 6443
+
+nodeSelector: {}
+
+priorityClassName: ""
+
+# Url to access prometheus
+prometheus:
+  url: http://prometheus.default.svc
+  port: 9090
+  path: ""
+
+replicas: 1
+
+rbac:
+  # Specifies whether RBAC resources should be created
+  create: true
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name:
+
+resources: {}
+  # requests:
+  #   cpu: 100m
+  #   memory: 128Mi
+  # limits:
+  #   cpu: 100m
+  #   memory: 128Mi
+
+rules:
+  default: true
+  custom: []
+# - seriesQuery: '{__name__=~"^some_metric_count$"}'
+#   resources:
+#     template: <<.Resource>>
+#   name:
+#     matches: ""
+#     as: "my_custom_metric"
+#   metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+  # Mounts a configMap with pre-generated rules for use. Overrides the
+  # default, custom, external and resource entries
+  existing:
+  external: []
+# - seriesQuery: '{__name__=~"^some_metric_count$"}'
+#   resources:
+#     template: <<.Resource>>
+#   name:
+#     matches: ""
+#     as: "my_external_metric"
+#   metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+  resource: {}
+#   cpu:
+#     containerQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>}[3m])) by (<<.GroupBy>>)
+#     nodeQuery: sum(rate(container_cpu_usage_seconds_total{<<.LabelMatchers>>, id='/'}[3m])) by (<<.GroupBy>>)
+#     resources:
+#       overrides:
+#         instance:
+#           resource: node
+#         namespace:
+#           resource: namespace
+#         pod:
+#           resource: pod
+#     containerLabel: container
+#   memory:
+#     containerQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>}) by (<<.GroupBy>>)
+#     nodeQuery: sum(container_memory_working_set_bytes{<<.LabelMatchers>>,id='/'}) by (<<.GroupBy>>)
+#     resources:
+#       overrides:
+#         instance:
+#           resource: node
+#         namespace:
+#           resource: namespace
+#         pod:
+#           resource: pod
+#     containerLabel: container
+#   window: 3m
+
+service:
+  annotations: {}
+  port: 443
+  type: ClusterIP
+
+tls:
+  enable: false
+  ca: |-
+    # Public CA file that signed the APIService
+  key: |-
+    # Private key of the APIService
+  certificate: |-
+    # Public key of the APIService
+
+# Any extra volumes
+extraVolumes: []
+  # - name: example-name
+  #   hostPath:
+  #     path: /path/on/host
+  #     type: DirectoryOrCreate
+  # - name: ssl-certs
+  #   hostPath:
+  #     path: /etc/ssl/certs/ca-bundle.crt
+  #     type: File
+
+# Any extra volume mounts
+extraVolumeMounts: []
+  #   - name: example-name
+  #     mountPath: /path/in/container
+  #   - name: ssl-certs
+  #     mountPath: /etc/ssl/certs/ca-certificates.crt
+  #     readOnly: true
+
+tolerations: []
+
+# Labels added to the pod
+podLabels: {}
+
+# Annotations added to the pod
+podAnnotations: {}
+
+hostNetwork:
+  # Specifies if prometheus-adapter should be started in hostNetwork mode.
+  #
+  # You would require this enabled if you use alternate overlay networking for pods and
+  # API server unable to communicate with metrics-server. As an example, this is required
+  # if you use Weave network on EKS
+  enabled: false
diff --git a/kubernetes/helm_charts/nginx-ingress/Chart.yaml b/kubernetes/helm_charts/nginx-ingress/Chart.yaml
new file mode 100644
index 0000000000..1aa28e1e29
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/Chart.yaml
@@ -0,0 +1,16 @@
+name: nginx-ingress
+version: 0.10.0
+appVersion: 1.12.0
+apiVersion: v1
+kubeVersion: ">= 1.16.0-0"
+description: NGINX Ingress Controller
+icon: https://raw.githubusercontent.com/nginxinc/kubernetes-ingress/v1.12.0/deployments/helm-chart/chart-icon.png
+home: https://github.com/nginxinc/kubernetes-ingress
+sources:
+  - https://github.com/nginxinc/kubernetes-ingress/tree/v1.12.0/deployments/helm-chart
+keywords:
+  - ingress
+  - nginx
+maintainers:
+  - name: nginxinc
+    email: kubernetes@nginx.com
diff --git a/kubernetes/helm_charts/nginx-ingress/README.md b/kubernetes/helm_charts/nginx-ingress/README.md
new file mode 100644
index 0000000000..d0f997f395
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/README.md
@@ -0,0 +1,230 @@
+# NGINX Ingress Controller Helm Chart
+
+## Introduction
+
+This chart deploys the NGINX Ingress controller in your Kubernetes cluster.
+
+## Prerequisites
+
+  - A [Kubernetes Version Supported by the Ingress Controller](https://docs.nginx.com/nginx-ingress-controller/technical-specifications/#supported-kubernetes-versions)
+  - Helm 3.0+.
+  - Git.
+  - If you’d like to use NGINX Plus:
+    - Build an Ingress controller image with NGINX Plus and push it to your private registry by following the instructions from [here](../../build/README.md).
+    - Update the `controller.image.repository` field of the `values-plus.yaml` accordingly.
+
+## Getting the Chart Sources
+
+This step is required if you're installing the chart using its sources. Additionally, the step is also required for managing the custom resource definitions (CRDs), which the Ingress Controller requires by default, or for upgrading/deleting the CRDs.
+
+1. Clone the Ingress controller repo:
+    ```console
+    $ git clone https://github.com/nginxinc/kubernetes-ingress/
+    ```
+2. Change your working directory to /deployments/helm-chart:
+    ```console
+    $ cd kubernetes-ingress/deployments/helm-chart
+    $ git checkout v1.12.0
+    ```
+
+## Adding the Helm Repository
+
+This step is required if you're installing the chart via the helm repository.
+
+```console
+$ helm repo add nginx-stable https://helm.nginx.com/stable
+$ helm repo update
+```
+
+**Note**: If you wish to use the experimental repository, replace `stable` with `edge`.
+
+## Installing the Chart
+
+### Installing the CRDs
+
+By default, the Ingress Controller requires a number of custom resource definitions (CRDs) installed in the cluster. The Helm client will install those CRDs.
+
+If you do not use the custom resources that require those CRDs (which corresponds to `controller.enableCustomResources` set to `false` and `controller.appprotect.enable` set to `false`). The installation of the CRDs can be skipped by specifying `--skip-crds` for the helm install command.
+
+### Installing via Helm Repository
+
+To install the chart with the release name my-release (my-release is the name that you choose):
+
+For NGINX:
+```console
+$ helm install my-release nginx-stable/nginx-ingress
+```
+
+For NGINX Plus: (assuming you have pushed the Ingress controller image `nginx-plus-ingress` to your private registry `myregistry.example.com`)
+```console
+$ helm install my-release nginx-stable/nginx-ingress --set controller.image.repository=myregistry.example.com/nginx-plus-ingress --set controller.nginxplus=true
+```
+
+**Note**: If you wish to use the experimental repository, replace `stable` with `edge` and add the `--devel` flag.
+
+### Installing Using Chart Sources
+
+To install the chart with the release name my-release (my-release is the name that you choose):
+
+For NGINX:
+```console
+$ helm install my-release .
+```
+
+For NGINX Plus:
+```console
+$ helm install my-release -f values-plus.yaml .
+```
+
+**Note**: If you wish to use the experimental repository, replace the value in the `tag` field inside the yaml files with `edge`.
+
+The command deploys the Ingress controller in your Kubernetes cluster in the default configuration. The configuration section lists the parameters that can be configured during installation.
+
+When deploying the Ingress controller, make sure to use your own TLS certificate and key for the default server rather than the default pre-generated ones. Read the [Configuration](#Configuration) section below to see how to configure a TLS certificate and key for the default server. Note that the default server returns the Not Found page with the 404 status code for all requests for domains for which there are no Ingress rules defined.
+
+## Upgrading the Chart
+
+### Upgrading the CRDs
+
+Helm does not upgrade the CRDs during a release upgrade. Before you upgrade a release, run the following command to upgrade the CRDs:
+
+```console
+$ kubectl apply -f crds/
+```
+> **Note**: The following warning is expected and can be ignored: `Warning: kubectl apply should be used on resource created by either kubectl create --save-config or kubectl apply`.
+
+> **Note**: Make sure to check the [release notes](https://www.github.com/nginxinc/kubernetes-ingress/releases) for a new release for any special upgrade procedures.
+
+### Upgrading the Release
+
+To upgrade the release `my-release`:
+
+#### Upgrade Using Chart Sources:
+
+```console
+$ helm upgrade my-release .
+```
+
+#### Upgrade via Helm Repository:
+
+```console
+$ helm upgrade my-release nginx-stable/nginx-ingress
+```
+
+## Uninstalling the Chart
+
+### Uninstalling the Release
+
+To uninstall/delete the release `my-release`:
+
+```console
+$ helm uninstall my-release
+```
+
+The command removes all the Kubernetes components associated with the release and deletes the release.
+
+### Uninstalling the CRDs
+
+Uninstalling the release does not remove the CRDs. To remove the CRDs, run:
+
+```console
+$ kubectl delete -f crds/
+```
+> **Note**: This command will delete all the corresponding custom resources in your cluster across all namespaces. Please ensure there are no custom resources that you want to keep and there are no other Ingress Controller releases running in the cluster.
+
+## Running Multiple Ingress Controllers
+
+If you are running multiple Ingress Controller releases in your cluster with enabled custom resources, the releases will share a single version of the CRDs. As a result, make sure that the Ingress Controller versions match the version of the CRDs. Additionally, when uninstalling a release, ensure that you don’t remove the CRDs until there are no other Ingress Controller releases running in the cluster.
+
+See [running multiple ingress controllers](https://docs.nginx.com/nginx-ingress-controller/installation/running-multiple-ingress-controllers/) for more details.
+
+## Configuration
+
+The following tables lists the configurable parameters of the NGINX Ingress controller chart and their default values.
+
+Parameter | Description | Default
+--- | --- | ---
+`controller.name` | The name of the Ingress controller daemonset or deployment. | Autogenerated
+`controller.kind` | The kind of the Ingress controller installation - deployment or daemonset. | deployment
+`controller.nginxplus` | Deploys the Ingress controller for NGINX Plus. | false
+`controller.nginxReloadTimeout` | The timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start. The default is 4000 (or 20000 if `controller.appprotect.enable` is true). If set to 0, the default value will be used. | 0
+`controller.hostNetwork` | Enables the Ingress controller pods to use the host's network namespace. | false
+`controller.nginxDebug` | Enables debugging for NGINX. Uses the `nginx-debug` binary. Requires `error-log-level: debug` in the ConfigMap via `controller.config.entries`. | false
+`controller.logLevel` | The log level of the Ingress Controller. | 1
+`controller.image.repository` | The image repository of the Ingress controller. | nginx/nginx-ingress
+`controller.image.tag` | The tag of the Ingress controller image. | 1.12.0
+`controller.image.pullPolicy` | The pull policy for the Ingress controller image. | IfNotPresent
+`controller.config.name` | The name of the ConfigMap used by the Ingress controller. | Autogenerated
+`controller.config.annotations` | The annotations of the Ingress controller configmap. | {}
+`controller.config.entries` | The entries of the ConfigMap for customizing NGINX configuration. See [ConfigMap resource docs](https://docs.nginx.com/nginx-ingress-controller/configuration/global-configuration/configmap-resource/) for the list of supported ConfigMap keys. | {}
+`controller.customPorts` | A list of custom ports to expose on the NGINX ingress controller pod. Follows the conventional Kubernetes yaml syntax for container ports. | []
+`controller.defaultTLS.cert` | The base64-encoded TLS certificate for the default HTTPS server. If not specified, a pre-generated self-signed certificate is used. **Note:** It is recommended that you specify your own certificate. | A pre-generated self-signed certificate.
+`controller.defaultTLS.key` | The base64-encoded TLS key for the default HTTPS server. **Note:** If not specified, a pre-generated key is used. It is recommended that you specify your own key. | A pre-generated key.
+`controller.defaultTLS.secret` | The secret with a TLS certificate and key for the default HTTPS server. The value must follow the following format: `<namespace>/<name>`. Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters. | None
+`controller.wildcardTLS.cert` | The base64-encoded TLS certificate for every Ingress host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. | None
+`controller.wildcardTLS.key` | The base64-encoded TLS key for every Ingress host that has TLS enabled but no secret specified. If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection. | None
+`controller.wildcardTLS.secret` | The secret with a TLS certificate and key for every Ingress host that has TLS enabled but no secret specified. The value must follow the following format: `<namespace>/<name>`. Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters. | None
+`controller.nodeSelector` | The node selector for pod assignment for the Ingress controller pods. | {}
+`controller.terminationGracePeriodSeconds` | The termination grace period of the Ingress controller pod. | 30
+`controller.tolerations` | The tolerations of the Ingress controller pods. | []
+`controller.affinity` | The affinity of the Ingress controller pods. | {}
+`controller.volumes` | The volumes of the Ingress controller pods. | []
+`controller.volumeMounts` | The volumeMounts of the Ingress controller pods. | []
+`controller.resources` | The resources of the Ingress controller pods. | {}
+`controller.replicaCount` | The number of replicas of the Ingress controller deployment. | 1
+`controller.ingressClass` | A class of the Ingress controller. For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise, the Ingress Controller will fail to start. The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class. For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class - i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources) or field "ingressClassName" (for VirtualServer/VirtualServerRoute/TransportServer resources) equal to the class. Additionally, the Ingress Controller processes resources that do not have the class set, which can be disabled by setting the `controller.useIngressClassOnly` parameter to `true`. The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes. | nginx
+`controller.useIngressClassOnly` | Ignore Ingress resources without the `"kubernetes.io/ingress.class"` annotation. For kubernetes versions >= 1.18 this flag will be IGNORED. | false
+`controller.setAsDefaultIngress` | New Ingresses without an `"ingressClassName"` field specified will be assigned the class specified in `controller.ingressClass`. Only for kubernetes versions >= 1.18. | false
+`controller.watchNamespace` | Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces. | ""
+`controller.enableCustomResources` | Enable the custom resources. | true
+`controller.enablePreviewPolicies` | Enable preview policies. | false
+`controller.enableTLSPassthrough` | Enable TLS Passthrough on port 443. Requires `controller.enableCustomResources`. | false
+`controller.globalConfiguration.create` | Creates the GlobalConfiguration custom resource. Requires `controller.enableCustomResources`. | false
+`controller.globalConfiguration.spec` | The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller. | {}
+`controller.enableSnippets` | Enable custom NGINX configuration snippets in VirtualServer, VirtualServerRoute and TransportServer resources. | false
+`controller.healthStatus` | Add a location "/nginx-health" to the default server. The location responds with the 200 status code for any request. Useful for external health-checking of the Ingress controller. | false
+`controller.healthStatusURI` | Sets the URI of health status location in the default server. Requires `controller.healthStatus`. | "/nginx-health"
+`controller.nginxStatus.enable` | Enable the NGINX stub_status, or the NGINX Plus API. | true
+`controller.nginxStatus.port` | Set the port where the NGINX stub_status or the NGINX Plus API is exposed. | 8080
+`controller.nginxStatus.allowCidrs` | Add IPv4 IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas. | 127.0.0.1
+`controller.priorityClassName` | The PriorityClass of the Ingress controller pods. | None
+`controller.service.create` | Creates a service to expose the Ingress controller pods. | true
+`controller.service.type` | The type of service to create for the Ingress controller. | LoadBalancer
+`controller.service.externalTrafficPolicy` | The externalTrafficPolicy of the service. The value Local preserves the client source IP. | Local
+`controller.service.annotations` | The annotations of the Ingress controller service. | {}
+`controller.service.extraLabels` | The extra labels of the service. | {}
+`controller.service.loadBalancerIP` | The static IP address for the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | ""
+`controller.service.externalIPs` | The list of external IPs for the Ingress controller service. | []
+`controller.service.loadBalancerSourceRanges` | The IP ranges (CIDR) that are allowed to access the load balancer. Requires `controller.service.type` set to `LoadBalancer`. The cloud provider must support this feature. | []
+`controller.service.name` | The name of the service. | Autogenerated
+`controller.service.customPorts` | A list of custom ports to expose through the Ingress controller service. Follows the conventional Kubernetes yaml syntax for service ports. | []
+`controller.service.httpPort.enable` | Enables the HTTP port for the Ingress controller service. | true
+`controller.service.httpPort.port` | The HTTP port of the Ingress controller service. | 80
+`controller.service.httpPort.nodePort` | The custom NodePort for the HTTP port. Requires `controller.service.type` set to `NodePort`. | ""
+`controller.service.httpPort.targetPort` | The target port of the HTTP port of the Ingress controller service. | 80
+`controller.service.httpsPort.enable` | Enables the HTTPS port for the Ingress controller service. | true
+`controller.service.httpsPort.port` | The HTTPS port of the Ingress controller service. | 443
+`controller.service.httpsPort.nodePort` | The custom NodePort for the HTTPS port. Requires `controller.service.type` set to `NodePort`.  | ""
+`controller.service.httpsPort.targetPort` | The target port of the HTTPS port of the Ingress controller service. | 443
+`controller.serviceAccount.name` | The name of the service account of the Ingress controller pods. Used for RBAC. | Autogenerated
+`controller.serviceAccount.imagePullSecretName` | The name of the secret containing docker registry credentials. Secret must exist in the same namespace as the helm release. | ""
+`controller.reportIngressStatus.enable` | Updates the address field in the status of Ingress resources with an external address of the Ingress controller. You must also specify the source of the external address either through an external service via `controller.reportIngressStatus.externalService`, `controller.reportIngressStatus.ingressLink` or the `external-status-address` entry in the ConfigMap via `controller.config.entries`. **Note:** `controller.config.entries.external-status-address` takes precedence over the others. | true
+`controller.reportIngressStatus.externalService` | Specifies the name of the service with the type LoadBalancer through which the Ingress controller is exposed externally. The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. The default is autogenerated and enabled when `controller.service.create` is set to `true` and `controller.service.type` is set to `LoadBalancer`. | Autogenerated
+`controller.reportIngressStatus.ingressLink` |  Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system. The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources. `controller.reportIngressStatus.enable` must be set to `true`. | ""
+`controller.reportIngressStatus.enableLeaderElection` | Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. `controller.reportIngressStatus.enable` must be set to `true`. | true
+`controller.reportIngressStatus.leaderElectionLockName` | Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true. | Autogenerated
+`controller.reportIngressStatus.annotations` | The annotations of the leader election configmap. | {}
+`controller.pod.annotations` | The annotations of the Ingress Controller pod. | {}
+`controller.appprotect.enable` | Enables the App Protect module in the Ingress Controller. | false
+`controller.readyStatus.enable` | Enables the readiness endpoint `"/nginx-ready"`. The endpoint returns a success code when NGINX has loaded all the config after the startup. This also configures a readiness probe for the Ingress Controller pods that uses the readiness endpoint. | true
+`controller.readyStatus.port` | The HTTP port for the readiness endpoint. | 8081
+`controller.enableLatencyMetrics` |  Enable collection of latency metrics for upstreams. Requires `prometheus.create`. | false
+`rbac.create` | Configures RBAC. | true
+`prometheus.create` | Expose NGINX or NGINX Plus metrics in the Prometheus format. | false
+`prometheus.port` | Configures the port to scrape the metrics. | 9113
+`prometheus.scheme` | Configures the HTTP scheme to use for connections to the Prometheus endpoint. | http
+`prometheus.secret` | The namespace / name of a Kubernetes TLS Secret. If specified, this secret is used to secure the Prometheus endpoint with TLS connections. | ""
+
+
+## Notes
+* The values-icp.yaml file is used for deploying the Ingress controller on IBM Cloud Private. See the [blog post](https://www.nginx.com/blog/nginx-ingress-controller-ibm-cloud-private/) for more details.
diff --git a/kubernetes/helm_charts/nginx-ingress/chart-icon.png b/kubernetes/helm_charts/nginx-ingress/chart-icon.png
new file mode 100644
index 0000000000..52961c9a6f
Binary files /dev/null and b/kubernetes/helm_charts/nginx-ingress/chart-icon.png differ
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml
new file mode 100644
index 0000000000..c06e46cb24
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_aplogconfs.yaml
@@ -0,0 +1,65 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+  creationTimestamp: null
+  name: aplogconfs.appprotect.f5.com
+spec:
+  group: appprotect.f5.com
+  names:
+    kind: APLogConf
+    listKind: APLogConfList
+    plural: aplogconfs
+    singular: aplogconf
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: APLogConf is the Schema for the APLogConfs API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: APLogConfSpec defines the desired state of APLogConf
+              properties:
+                content:
+                  properties:
+                    format:
+                      enum:
+                        - splunk
+                        - arcsight
+                        - default
+                        - user-defined
+                        - grpc
+                      type: string
+                    format_string:
+                      type: string
+                    max_message_size:
+                      pattern: ^([1-9]|[1-5][0-9]|6[0-4])k$
+                      type: string
+                    max_request_size:
+                      pattern: ^([1-9]|[1-9][0-9]|[1-9][0-9]{2}|1[0-9]{3}|20[1-3][0-9]|204[1-8]|any)$
+                      type: string
+                  type: object
+                filter:
+                  properties:
+                    request_type:
+                      enum:
+                        - all
+                        - illegal
+                        - blocked
+                      type: string
+                  type: object
+              type: object
+          type: object
+      served: true
+      storage: true
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml
new file mode 100644
index 0000000000..b4309db1a3
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_appolicies.yaml
@@ -0,0 +1,1790 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+  creationTimestamp: null
+  name: appolicies.appprotect.f5.com
+spec:
+  group: appprotect.f5.com
+  names:
+    kind: APPolicy
+    listKind: APPolicyList
+    plural: appolicies
+    singular: appolicy
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: APPolicyConfig is the Schema for the APPolicyconfigs API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: APPolicySpec defines the desired state of APPolicy
+              properties:
+                modifications:
+                  items:
+                    properties:
+                      action:
+                        type: string
+                      description:
+                        type: string
+                      entity:
+                        properties:
+                          name:
+                            type: string
+                        type: object
+                      entityChanges:
+                        properties:
+                          type:
+                            type: string
+                        type: object
+                    type: object
+                    x-kubernetes-preserve-unknown-fields: true
+                  type: array
+                modificationsReference:
+                  properties:
+                    link:
+                      pattern: ^http
+                      type: string
+                  type: object
+                policy:
+                  description: Defines the App Protect policy
+                  properties:
+                    applicationLanguage:
+                      enum:
+                        - iso-8859-10
+                        - iso-8859-6
+                        - windows-1255
+                        - auto-detect
+                        - koi8-r
+                        - gb18030
+                        - iso-8859-8
+                        - windows-1250
+                        - iso-8859-9
+                        - windows-1252
+                        - iso-8859-16
+                        - gb2312
+                        - iso-8859-2
+                        - iso-8859-5
+                        - windows-1257
+                        - windows-1256
+                        - iso-8859-13
+                        - windows-874
+                        - windows-1253
+                        - iso-8859-3
+                        - euc-jp
+                        - utf-8
+                        - gbk
+                        - windows-1251
+                        - big5
+                        - iso-8859-1
+                        - shift_jis
+                        - euc-kr
+                        - iso-8859-4
+                        - iso-8859-7
+                        - iso-8859-15
+                      type: string
+                    blocking-settings:
+                      properties:
+                        evasions:
+                          items:
+                            properties:
+                              description:
+                                enum:
+                                  - '%u decoding'
+                                  - Apache whitespace
+                                  - Bad unescape
+                                  - Bare byte decoding
+                                  - Directory traversals
+                                  - IIS backslashes
+                                  - IIS Unicode codepoints
+                                  - Multiple decoding
+                                type: string
+                              enabled:
+                                type: boolean
+                              maxDecodingPasses:
+                                type: integer
+                            type: object
+                          type: array
+                        http-protocols:
+                          items:
+                            properties:
+                              description:
+                                enum:
+                                  - Unescaped space in URL
+                                  - Unparsable request content
+                                  - Several Content-Length headers
+                                  - 'POST request with Content-Length: 0'
+                                  - Null in request
+                                  - No Host header in HTTP/1.1 request
+                                  - Multiple host headers
+                                  - Host header contains IP address
+                                  - High ASCII characters in headers
+                                  - Header name with no header value
+                                  - CRLF characters before request start
+                                  - Content length should be a positive number
+                                  - Chunked request with Content-Length header
+                                  - Check maximum number of parameters
+                                  - Check maximum number of headers
+                                  - Body in GET or HEAD requests
+                                  - Bad multipart/form-data request parsing
+                                  - Bad multipart parameters parsing
+                                  - Bad HTTP version
+                                  - Bad host header value
+                                type: string
+                              enabled:
+                                type: boolean
+                              maxHeaders:
+                                type: integer
+                              maxParams:
+                                type: integer
+                            type: object
+                          type: array
+                        violations:
+                          items:
+                            properties:
+                              alarm:
+                                type: boolean
+                              block:
+                                type: boolean
+                              description:
+                                type: string
+                              name:
+                                enum:
+                                  - VIOL_GRPC_FORMAT
+                                  - VIOL_GRPC_MALFORMED
+                                  - VIOL_GRPC_METHOD
+                                  - VIOL_PARAMETER_ARRAY_VALUE
+                                  - VIOL_PARAMETER_VALUE_REGEXP
+                                  - VIOL_CSRF
+                                  - VIOL_PARAMETER_VALUE_BASE64
+                                  - VIOL_MANDATORY_HEADER
+                                  - VIOL_HEADER_REPEATED
+                                  - VIOL_ASM_COOKIE_MODIFIED
+                                  - VIOL_BLACKLISTED_IP
+                                  - VIOL_COOKIE_EXPIRED
+                                  - VIOL_COOKIE_LENGTH
+                                  - VIOL_COOKIE_MALFORMED
+                                  - VIOL_COOKIE_MODIFIED
+                                  - VIOL_DATA_GUARD
+                                  - VIOL_ENCODING
+                                  - VIOL_EVASION
+                                  - VIOL_FILETYPE
+                                  - VIOL_FILE_UPLOAD
+                                  - VIOL_FILE_UPLOAD_IN_BODY
+                                  - VIOL_HEADER_LENGTH
+                                  - VIOL_HEADER_METACHAR
+                                  - VIOL_HTTP_PROTOCOL
+                                  - VIOL_HTTP_RESPONSE_STATUS
+                                  - VIOL_JSON_FORMAT
+                                  - VIOL_JSON_MALFORMED
+                                  - VIOL_JSON_SCHEMA
+                                  - VIOL_MANDATORY_PARAMETER
+                                  - VIOL_MANDATORY_REQUEST_BODY
+                                  - VIOL_METHOD
+                                  - VIOL_PARAMETER
+                                  - VIOL_PARAMETER_DATA_TYPE
+                                  - VIOL_PARAMETER_EMPTY_VALUE
+                                  - VIOL_PARAMETER_LOCATION
+                                  - VIOL_PARAMETER_MULTIPART_NULL_VALUE
+                                  - VIOL_PARAMETER_NAME_METACHAR
+                                  - VIOL_PARAMETER_NUMERIC_VALUE
+                                  - VIOL_PARAMETER_REPEATED
+                                  - VIOL_PARAMETER_STATIC_VALUE
+                                  - VIOL_PARAMETER_VALUE_LENGTH
+                                  - VIOL_PARAMETER_VALUE_METACHAR
+                                  - VIOL_POST_DATA_LENGTH
+                                  - VIOL_QUERY_STRING_LENGTH
+                                  - VIOL_RATING_THREAT
+                                  - VIOL_RATING_NEED_EXAMINATION
+                                  - VIOL_REQUEST_MAX_LENGTH
+                                  - VIOL_REQUEST_LENGTH
+                                  - VIOL_THREAT_CAMPAIGN
+                                  - VIOL_URL
+                                  - VIOL_URL_CONTENT_TYPE
+                                  - VIOL_URL_LENGTH
+                                  - VIOL_URL_METACHAR
+                                  - VIOL_XML_FORMAT
+                                  - VIOL_XML_MALFORMED
+                                type: string
+                            type: object
+                          type: array
+                      type: object
+                    blockingSettingReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    bot-defense:
+                      properties:
+                        mitigations:
+                          properties:
+                            anomalies:
+                              items:
+                                properties:
+                                  action:
+                                    enum:
+                                      - alarm
+                                      - block
+                                      - default
+                                      - detect
+                                      - ignore
+                                    type: string
+                                  name:
+                                    type: string
+                                  scoreThreshold:
+                                    pattern: '[0-9]|[1-9][0-9]|1[0-4][0-9]|150|default'
+                                    type: string
+                                type: object
+                              type: array
+                            browsers:
+                              items:
+                                properties:
+                                  $action:
+                                    enum:
+                                      - delete
+                                    type: string
+                                  action:
+                                    enum:
+                                      - alarm
+                                      - block
+                                      - detect
+                                    type: string
+                                  maxVersion:
+                                    maximum: 2147483647
+                                    minimum: 0
+                                    type: integer
+                                  minVersion:
+                                    maximum: 2147483647
+                                    minimum: 0
+                                    type: integer
+                                  name:
+                                    type: string
+                                type: object
+                              type: array
+                            classes:
+                              items:
+                                properties:
+                                  action:
+                                    enum:
+                                      - alarm
+                                      - block
+                                      - detect
+                                      - ignore
+                                    type: string
+                                  name:
+                                    enum:
+                                      - browser
+                                      - malicious-bot
+                                      - suspicious-browser
+                                      - trusted-bot
+                                      - unknown
+                                      - untrusted-bot
+                                    type: string
+                                type: object
+                              type: array
+                            signatures:
+                              items:
+                                properties:
+                                  action:
+                                    enum:
+                                      - alarm
+                                      - block
+                                      - detect
+                                      - ignore
+                                    type: string
+                                  name:
+                                    type: string
+                                type: object
+                              type: array
+                          type: object
+                        settings:
+                          properties:
+                            isEnabled:
+                              type: boolean
+                          type: object
+                      type: object
+                    browser-definitions:
+                      items:
+                        properties:
+                          $action:
+                            enum:
+                              - delete
+                            type: string
+                          isUserDefined:
+                            type: boolean
+                          matchRegex:
+                            type: string
+                          matchString:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    caseInsensitive:
+                      type: boolean
+                    character-sets:
+                      items:
+                        properties:
+                          characterSet:
+                            items:
+                              properties:
+                                isAllowed:
+                                  type: boolean
+                                metachar:
+                                  type: string
+                              type: object
+                            type: array
+                          characterSetType:
+                            enum:
+                              - gwt-content
+                              - header
+                              - json-content
+                              - parameter-name
+                              - parameter-value
+                              - plain-text-content
+                              - url
+                              - xml-content
+                            type: string
+                        type: object
+                      type: array
+                    characterSetReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    cookie-settings:
+                      properties:
+                        maximumCookieHeaderLength:
+                          pattern: any|\d+
+                          type: string
+                      type: object
+                    cookieReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    cookieSettingsReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    cookies:
+                      items:
+                        properties:
+                          accessibleOnlyThroughTheHttpProtocol:
+                            type: boolean
+                          attackSignaturesCheck:
+                            type: boolean
+                          decodeValueAsBase64:
+                            enum:
+                              - enabled
+                              - disabled
+                              - required
+                            type: string
+                          enforcementType:
+                            type: string
+                          insertSameSiteAttribute:
+                            enum:
+                              - lax
+                              - none
+                              - none-value
+                              - strict
+                            type: string
+                          name:
+                            type: string
+                          securedOverHttpsConnection:
+                            type: boolean
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                          type:
+                            enum:
+                              - explicit
+                              - wildcard
+                            type: string
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    csrf-protection:
+                      properties:
+                        enabled:
+                          type: boolean
+                        expirationTimeInSeconds:
+                          pattern: disabled|\d+
+                          type: string
+                        sslOnly:
+                          type: boolean
+                      type: object
+                    csrf-urls:
+                      items:
+                        properties:
+                          $action:
+                            enum:
+                              - delete
+                            type: string
+                          enforcementAction:
+                            enum:
+                              - verify-origin
+                              - none
+                            type: string
+                          method:
+                            enum:
+                              - GET
+                              - POST
+                              - any
+                            type: string
+                          url:
+                            type: string
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    data-guard:
+                      properties:
+                        creditCardNumbers:
+                          type: boolean
+                        enabled:
+                          type: boolean
+                        enforcementMode:
+                          enum:
+                            - ignore-urls-in-list
+                            - enforce-urls-in-list
+                          type: string
+                        enforcementUrls:
+                          items:
+                            type: string
+                          type: array
+                        lastCcnDigitsToExpose:
+                          type: integer
+                        lastSsnDigitsToExpose:
+                          type: integer
+                        maskData:
+                          type: boolean
+                        usSocialSecurityNumbers:
+                          type: boolean
+                      type: object
+                    dataGuardReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    description:
+                      type: string
+                    enablePassiveMode:
+                      type: boolean
+                    enforcementMode:
+                      enum:
+                        - transparent
+                        - blocking
+                      type: string
+                    filetypeReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    filetypes:
+                      items:
+                        properties:
+                          action:
+                            type: string
+                          allowed:
+                            type: boolean
+                          checkPostDataLength:
+                            type: boolean
+                          checkQueryStringLength:
+                            type: boolean
+                          checkRequestLength:
+                            type: boolean
+                          checkUrlLength:
+                            type: boolean
+                          name:
+                            type: string
+                          postDataLength:
+                            type: integer
+                          queryStringLength:
+                            type: integer
+                          requestLength:
+                            type: integer
+                          responseCheck:
+                            type: boolean
+                          type:
+                            enum:
+                              - explicit
+                              - wildcard
+                            type: string
+                          urlLength:
+                            type: integer
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    fullPath:
+                      type: string
+                    general:
+                      properties:
+                        allowedResponseCodes:
+                          items:
+                            format: int32
+                            maximum: 999
+                            minimum: 100
+                            type: integer
+                          type: array
+                        customXffHeaders:
+                          items:
+                            type: string
+                          type: array
+                        maskCreditCardNumbersInRequest:
+                          type: boolean
+                        trustXff:
+                          type: boolean
+                      type: object
+                    generalReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    grpc-profiles:
+                      items:
+                        properties:
+                          associateUrls:
+                            type: boolean
+                          attackSignaturesCheck:
+                            type: boolean
+                          defenseAttributes:
+                            properties:
+                              allowUnknownFields:
+                                type: boolean
+                              maximumDataLength:
+                                pattern: any|\d+
+                                type: string
+                            type: object
+                          description:
+                            type: string
+                          hasIdlFiles:
+                            type: boolean
+                          idlFiles:
+                            items:
+                              properties:
+                                idlFile:
+                                  properties:
+                                    contents:
+                                      type: string
+                                    fileName:
+                                      type: string
+                                    isBase64:
+                                      type: boolean
+                                  type: object
+                                isPrimary:
+                                  type: boolean
+                                primaryIdlFileName:
+                                  type: string
+                              type: object
+                            type: array
+                          metacharElementCheck:
+                            type: boolean
+                          name:
+                            type: string
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                        type: object
+                      type: array
+                    header-settings:
+                      properties:
+                        maximumHttpHeaderLength:
+                          pattern: any|\d+
+                          type: string
+                      type: object
+                    headerReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    headerSettingsReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    headers:
+                      items:
+                        properties:
+                          allowRepeatedOccurrences:
+                            type: boolean
+                          base64Decoding:
+                            type: boolean
+                          checkSignatures:
+                            type: boolean
+                          decodeValueAsBase64:
+                            enum:
+                              - enabled
+                              - disabled
+                              - required
+                            type: string
+                          htmlNormalization:
+                            type: boolean
+                          mandatory:
+                            type: boolean
+                          maskValueInLogs:
+                            type: boolean
+                          name:
+                            type: string
+                          normalizationViolations:
+                            type: boolean
+                          percentDecoding:
+                            type: boolean
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                          type:
+                            enum:
+                              - explicit
+                              - wildcard
+                            type: string
+                          urlNormalization:
+                            type: boolean
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    host-names:
+                      items:
+                        properties:
+                          $action:
+                            enum:
+                              - delete
+                            type: string
+                          includeSubdomains:
+                            type: boolean
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    idl-files:
+                      items:
+                        properties:
+                          contents:
+                            type: string
+                          fileName:
+                            type: string
+                          isBase64:
+                            type: boolean
+                        type: object
+                      type: array
+                    json-profiles:
+                      items:
+                        properties:
+                          attackSignaturesCheck:
+                            type: boolean
+                          defenseAttributes:
+                            properties:
+                              maximumArrayLength:
+                                pattern: any|\d+
+                                type: string
+                              maximumStructureDepth:
+                                pattern: any|\d+
+                                type: string
+                              maximumTotalLengthOfJSONData:
+                                pattern: any|\d+
+                                type: string
+                              maximumValueLength:
+                                pattern: any|\d+
+                                type: string
+                              tolerateJSONParsingWarnings:
+                                type: boolean
+                            type: object
+                          description:
+                            type: string
+                          handleJsonValuesAsParameters:
+                            type: boolean
+                          hasValidationFiles:
+                            type: boolean
+                          metacharOverrides:
+                            items:
+                              properties:
+                                isAllowed:
+                                  type: boolean
+                                metachar:
+                                  type: string
+                              type: object
+                            type: array
+                          name:
+                            type: string
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                          validationFiles:
+                            items:
+                              properties:
+                                importUrl:
+                                  type: string
+                                isPrimary:
+                                  type: boolean
+                                jsonValidationFile:
+                                  properties:
+                                    contents:
+                                      type: string
+                                    fileName:
+                                      type: string
+                                    isBase64:
+                                      type: boolean
+                                  type: object
+                              type: object
+                            type: array
+                        type: object
+                      type: array
+                    json-validation-files:
+                      items:
+                        properties:
+                          contents:
+                            type: string
+                          fileName:
+                            type: string
+                          isBase64:
+                            type: boolean
+                        type: object
+                      type: array
+                    jsonProfileReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    jsonValidationFileReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    methodReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    methods:
+                      items:
+                        properties:
+                          action:
+                            type: string
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    name:
+                      type: string
+                    open-api-files:
+                      items:
+                        properties:
+                          link:
+                            pattern: ^http
+                            type: string
+                        type: object
+                      type: array
+                    parameterReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    parameters:
+                      items:
+                        properties:
+                          allowEmptyValue:
+                            type: boolean
+                          allowRepeatedParameterName:
+                            type: boolean
+                          arraySerializationFormat:
+                            enum:
+                              - csv
+                              - form
+                              - label
+                              - matrix
+                              - multi
+                              - multipart
+                              - pipe
+                              - ssv
+                              - tsv
+                            type: string
+                          attackSignaturesCheck:
+                            type: boolean
+                          checkMaxValue:
+                            type: boolean
+                          checkMaxValueLength:
+                            type: boolean
+                          checkMetachars:
+                            type: boolean
+                          checkMinValue:
+                            type: boolean
+                          checkMinValueLength:
+                            type: boolean
+                          checkMultipleOfValue:
+                            type: boolean
+                          contentProfile:
+                            properties:
+                              name:
+                                type: string
+                            type: object
+                          dataType:
+                            enum:
+                              - alpha-numeric
+                              - binary
+                              - boolean
+                              - decimal
+                              - email
+                              - integer
+                              - none
+                              - phone
+                            type: string
+                          decodeValueAsBase64:
+                            enum:
+                              - enabled
+                              - disabled
+                              - required
+                            type: string
+                          disallowFileUploadOfExecutables:
+                            type: boolean
+                          enableRegularExpression:
+                            type: boolean
+                          exclusiveMax:
+                            type: boolean
+                          exclusiveMin:
+                            type: boolean
+                          isBase64:
+                            type: boolean
+                          isCookie:
+                            type: boolean
+                          isHeader:
+                            type: boolean
+                          level:
+                            enum:
+                              - global
+                              - url
+                            type: string
+                          mandatory:
+                            type: boolean
+                          maximumLength:
+                            type: integer
+                          maximumValue:
+                            type: integer
+                          metacharsOnParameterValueCheck:
+                            type: boolean
+                          minimumLength:
+                            type: integer
+                          minimumValue:
+                            type: integer
+                          multipleOf:
+                            type: integer
+                          name:
+                            type: string
+                          nameMetacharOverrides:
+                            items:
+                              properties:
+                                isAllowed:
+                                  type: boolean
+                                metachar:
+                                  type: string
+                              type: object
+                            type: array
+                          objectSerializationStyle:
+                            type: string
+                          parameterEnumValues:
+                            items:
+                              type: string
+                            type: array
+                          parameterLocation:
+                            enum:
+                              - any
+                              - cookie
+                              - form-data
+                              - header
+                              - path
+                              - query
+                            type: string
+                          regularExpression:
+                            type: string
+                          sensitiveParameter:
+                            type: boolean
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                          staticValues:
+                            type: string
+                          type:
+                            enum:
+                              - explicit
+                              - wildcard
+                            type: string
+                          url:
+                            type: object
+                          valueMetacharOverrides:
+                            items:
+                              properties:
+                                isAllowed:
+                                  type: boolean
+                                metachar:
+                                  type: string
+                              type: object
+                            type: array
+                          valueType:
+                            enum:
+                              - array
+                              - auto-detect
+                              - dynamic-content
+                              - dynamic-parameter-name
+                              - ignore
+                              - json
+                              - object
+                              - openapi-array
+                              - static-content
+                              - user-input
+                              - xml
+                            type: string
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    response-pages:
+                      items:
+                        properties:
+                          ajaxActionType:
+                            enum:
+                              - alert-popup
+                              - custom
+                              - redirect
+                            type: string
+                          ajaxCustomContent:
+                            type: string
+                          ajaxEnabled:
+                            type: boolean
+                          ajaxPopupMessage:
+                            type: string
+                          ajaxRedirectUrl:
+                            type: string
+                          grpcStatusCode:
+                            pattern: ABORTED|ALREADY_EXISTS|CANCELLED|DATA_LOSS|DEADLINE_EXCEEDED|FAILED_PRECONDITION|INTERNAL|INVALID_ARGUMENT|NOT_FOUND|OK|OUT_OF_RANGE|PERMISSION_DENIED|RESOURCE_EXHAUSTED|UNAUTHENTICATED|UNAVAILABLE|UNIMPLEMENTED|UNKNOWN|d+
+                            type: string
+                          grpcStatusMessage:
+                            type: string
+                          responseActionType:
+                            enum:
+                              - custom
+                              - default
+                              - erase-cookies
+                              - redirect
+                              - soap-fault
+                            type: string
+                          responseContent:
+                            type: string
+                          responseHeader:
+                            type: string
+                          responsePageType:
+                            enum:
+                              - ajax
+                              - ajax-login
+                              - captcha
+                              - captcha-fail
+                              - default
+                              - failed-login-honeypot
+                              - failed-login-honeypot-ajax
+                              - hijack
+                              - leaked-credentials
+                              - leaked-credentials-ajax
+                              - mobile
+                              - persistent-flow
+                              - xml
+                              - grpc
+                            type: string
+                          responseRedirectUrl:
+                            type: string
+                        type: object
+                      type: array
+                    responsePageReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    sensitive-parameters:
+                      items:
+                        properties:
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    sensitiveParameterReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    server-technologies:
+                      items:
+                        properties:
+                          serverTechnologyName:
+                            enum:
+                              - Jenkins
+                              - SharePoint
+                              - Oracle Application Server
+                              - Python
+                              - Oracle Identity Manager
+                              - Spring Boot
+                              - CouchDB
+                              - SQLite
+                              - Handlebars
+                              - Mustache
+                              - Prototype
+                              - Zend
+                              - Redis
+                              - Underscore.js
+                              - Ember.js
+                              - ZURB Foundation
+                              - ef.js
+                              - Vue.js
+                              - UIKit
+                              - TYPO3 CMS
+                              - RequireJS
+                              - React
+                              - MooTools
+                              - Laravel
+                              - GraphQL
+                              - Google Web Toolkit
+                              - Express.js
+                              - CodeIgniter
+                              - Backbone.js
+                              - AngularJS
+                              - JavaScript
+                              - Nginx
+                              - Jetty
+                              - Joomla
+                              - JavaServer Faces (JSF)
+                              - Ruby
+                              - MongoDB
+                              - Django
+                              - Node.js
+                              - Citrix
+                              - JBoss
+                              - Elasticsearch
+                              - Apache Struts
+                              - XML
+                              - PostgreSQL
+                              - IBM DB2
+                              - Sybase/ASE
+                              - CGI
+                              - Proxy Servers
+                              - SSI (Server Side Includes)
+                              - Cisco
+                              - Novell
+                              - Macromedia JRun
+                              - BEA Systems WebLogic Server
+                              - Lotus Domino
+                              - MySQL
+                              - Oracle
+                              - Microsoft SQL Server
+                              - PHP
+                              - Outlook Web Access
+                              - Apache/NCSA HTTP Server
+                              - Apache Tomcat
+                              - WordPress
+                              - Macromedia ColdFusion
+                              - Unix/Linux
+                              - Microsoft Windows
+                              - ASP.NET
+                              - Front Page Server Extensions (FPSE)
+                              - IIS
+                              - WebDAV
+                              - ASP
+                              - Java Servlets/JSP
+                              - jQuery
+                            type: string
+                        type: object
+                      type: array
+                    serverTechnologyReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    signature-requirements:
+                      items:
+                        properties:
+                          tag:
+                            type: string
+                        type: object
+                      type: array
+                    signature-sets:
+                      items:
+                        properties:
+                          alarm:
+                            type: boolean
+                          block:
+                            type: boolean
+                          name:
+                            type: string
+                        type: object
+                        x-kubernetes-preserve-unknown-fields: true
+                      type: array
+                    signature-settings:
+                      properties:
+                        attackSignatureFalsePositiveMode:
+                          enum:
+                            - detect
+                            - detect-and-allow
+                            - disabled
+                          type: string
+                        minimumAccuracyForAutoAddedSignatures:
+                          enum:
+                            - high
+                            - low
+                            - medium
+                          type: string
+                      type: object
+                    signatureReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    signatureSetReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    signatureSettingReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    signatures:
+                      items:
+                        properties:
+                          enabled:
+                            type: boolean
+                          name:
+                            type: string
+                          signatureId:
+                            type: integer
+                          tag:
+                            type: string
+                        type: object
+                      type: array
+                    softwareVersion:
+                      type: string
+                    template:
+                      properties:
+                        name:
+                          type: string
+                      type: object
+                    threat-campaigns:
+                      items:
+                        properties:
+                          isEnabled:
+                            type: boolean
+                          name:
+                            type: string
+                        type: object
+                      type: array
+                    threatCampaignReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    urlReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    urls:
+                      items:
+                        properties:
+                          $action:
+                            enum:
+                              - delete
+                            type: string
+                          allowRenderingInFrames:
+                            enum:
+                              - never
+                              - only-same
+                            type: string
+                          allowRenderingInFramesOnlyFrom:
+                            type: string
+                          attackSignaturesCheck:
+                            type: boolean
+                          clickjackingProtection:
+                            type: boolean
+                          description:
+                            type: string
+                          disallowFileUploadOfExecutables:
+                            type: boolean
+                          html5CrossOriginRequestsEnforcement:
+                            properties:
+                              allowOriginsEnforcementMode:
+                                enum:
+                                  - replace-with
+                                  - unmodified
+                                type: string
+                              checkAllowedMethods:
+                                type: boolean
+                              crossDomainAllowedOrigin:
+                                items:
+                                  properties:
+                                    includeSubDomains:
+                                      type: boolean
+                                    originName:
+                                      type: string
+                                    originPort:
+                                      pattern: any|\d+
+                                      type: string
+                                    originProtocol:
+                                      enum:
+                                        - http
+                                        - http/https
+                                        - https
+                                      type: string
+                                  type: object
+                                type: array
+                              enforcementMode:
+                                enum:
+                                  - disabled
+                                  - enforce
+                                type: string
+                            type: object
+                          isAllowed:
+                            type: boolean
+                          mandatoryBody:
+                            type: boolean
+                          metacharOverrides:
+                            items:
+                              properties:
+                                isAllowed:
+                                  type: boolean
+                                metachar:
+                                  type: string
+                              type: object
+                            type: array
+                          metacharsOnUrlCheck:
+                            type: boolean
+                          method:
+                            enum:
+                              - ACL
+                              - BCOPY
+                              - BDELETE
+                              - BMOVE
+                              - BPROPFIND
+                              - BPROPPATCH
+                              - CHECKIN
+                              - CHECKOUT
+                              - CONNECT
+                              - COPY
+                              - DELETE
+                              - GET
+                              - HEAD
+                              - LINK
+                              - LOCK
+                              - MERGE
+                              - MKCOL
+                              - MKWORKSPACE
+                              - MOVE
+                              - NOTIFY
+                              - OPTIONS
+                              - PATCH
+                              - POLL
+                              - POST
+                              - PROPFIND
+                              - PROPPATCH
+                              - PUT
+                              - REPORT
+                              - RPC_IN_DATA
+                              - RPC_OUT_DATA
+                              - SEARCH
+                              - SUBSCRIBE
+                              - TRACE
+                              - TRACK
+                              - UNLINK
+                              - UNLOCK
+                              - UNSUBSCRIBE
+                              - VERSION_CONTROL
+                              - X-MS-ENUMATTS
+                              - '*'
+                            type: string
+                          methodOverrides:
+                            items:
+                              properties:
+                                allowed:
+                                  type: boolean
+                                method:
+                                  enum:
+                                    - ACL
+                                    - BCOPY
+                                    - BDELETE
+                                    - BMOVE
+                                    - BPROPFIND
+                                    - BPROPPATCH
+                                    - CHECKIN
+                                    - CHECKOUT
+                                    - CONNECT
+                                    - COPY
+                                    - DELETE
+                                    - GET
+                                    - HEAD
+                                    - LINK
+                                    - LOCK
+                                    - MERGE
+                                    - MKCOL
+                                    - MKWORKSPACE
+                                    - MOVE
+                                    - NOTIFY
+                                    - OPTIONS
+                                    - PATCH
+                                    - POLL
+                                    - POST
+                                    - PROPFIND
+                                    - PROPPATCH
+                                    - PUT
+                                    - REPORT
+                                    - RPC_IN_DATA
+                                    - RPC_OUT_DATA
+                                    - SEARCH
+                                    - SUBSCRIBE
+                                    - TRACE
+                                    - TRACK
+                                    - UNLINK
+                                    - UNLOCK
+                                    - UNSUBSCRIBE
+                                    - VERSION_CONTROL
+                                    - X-MS-ENUMATTS
+                                  type: string
+                              type: object
+                            type: array
+                          methodsOverrideOnUrlCheck:
+                            type: boolean
+                          name:
+                            type: string
+                          operationId:
+                            type: string
+                          positionalParameters:
+                            items:
+                              properties:
+                                parameter:
+                                  properties:
+                                    allowEmptyValue:
+                                      type: boolean
+                                    allowRepeatedParameterName:
+                                      type: boolean
+                                    arraySerializationFormat:
+                                      enum:
+                                        - csv
+                                        - form
+                                        - label
+                                        - matrix
+                                        - multi
+                                        - multipart
+                                        - pipe
+                                        - ssv
+                                        - tsv
+                                      type: string
+                                    attackSignaturesCheck:
+                                      type: boolean
+                                    checkMaxValue:
+                                      type: boolean
+                                    checkMaxValueLength:
+                                      type: boolean
+                                    checkMetachars:
+                                      type: boolean
+                                    checkMinValue:
+                                      type: boolean
+                                    checkMinValueLength:
+                                      type: boolean
+                                    checkMultipleOfValue:
+                                      type: boolean
+                                    contentProfile:
+                                      properties:
+                                        name:
+                                          type: string
+                                      type: object
+                                    dataType:
+                                      enum:
+                                        - alpha-numeric
+                                        - binary
+                                        - boolean
+                                        - decimal
+                                        - email
+                                        - integer
+                                        - none
+                                        - phone
+                                      type: string
+                                    decodeValueAsBase64:
+                                      enum:
+                                        - enabled
+                                        - disabled
+                                        - required
+                                      type: string
+                                    disallowFileUploadOfExecutables:
+                                      type: boolean
+                                    enableRegularExpression:
+                                      type: boolean
+                                    exclusiveMax:
+                                      type: boolean
+                                    exclusiveMin:
+                                      type: boolean
+                                    isBase64:
+                                      type: boolean
+                                    isCookie:
+                                      type: boolean
+                                    isHeader:
+                                      type: boolean
+                                    level:
+                                      enum:
+                                        - global
+                                        - url
+                                      type: string
+                                    mandatory:
+                                      type: boolean
+                                    maximumLength:
+                                      type: integer
+                                    maximumValue:
+                                      type: integer
+                                    metacharsOnParameterValueCheck:
+                                      type: boolean
+                                    minimumLength:
+                                      type: integer
+                                    minimumValue:
+                                      type: integer
+                                    multipleOf:
+                                      type: integer
+                                    name:
+                                      type: string
+                                    nameMetacharOverrides:
+                                      items:
+                                        properties:
+                                          isAllowed:
+                                            type: boolean
+                                          metachar:
+                                            type: string
+                                        type: object
+                                      type: array
+                                    objectSerializationStyle:
+                                      type: string
+                                    parameterEnumValues:
+                                      items:
+                                        type: string
+                                      type: array
+                                    parameterLocation:
+                                      enum:
+                                        - any
+                                        - cookie
+                                        - form-data
+                                        - header
+                                        - path
+                                        - query
+                                      type: string
+                                    regularExpression:
+                                      type: string
+                                    sensitiveParameter:
+                                      type: boolean
+                                    signatureOverrides:
+                                      items:
+                                        properties:
+                                          enabled:
+                                            type: boolean
+                                          name:
+                                            type: string
+                                          signatureId:
+                                            type: integer
+                                          tag:
+                                            type: string
+                                        type: object
+                                      type: array
+                                    staticValues:
+                                      type: string
+                                    type:
+                                      enum:
+                                        - explicit
+                                        - wildcard
+                                      type: string
+                                    url:
+                                      type: object
+                                    valueMetacharOverrides:
+                                      items:
+                                        properties:
+                                          isAllowed:
+                                            type: boolean
+                                          metachar:
+                                            type: string
+                                        type: object
+                                      type: array
+                                    valueType:
+                                      enum:
+                                        - array
+                                        - auto-detect
+                                        - dynamic-content
+                                        - dynamic-parameter-name
+                                        - ignore
+                                        - json
+                                        - object
+                                        - openapi-array
+                                        - static-content
+                                        - user-input
+                                        - xml
+                                      type: string
+                                    wildcardOrder:
+                                      type: integer
+                                  type: object
+                                urlSegmentIndex:
+                                  type: integer
+                              type: object
+                            type: array
+                          protocol:
+                            enum:
+                              - http
+                              - https
+                            type: string
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                          type:
+                            enum:
+                              - explicit
+                              - wildcard
+                            type: string
+                          urlContentProfiles:
+                            items:
+                              properties:
+                                contentProfile:
+                                  properties:
+                                    name:
+                                      type: string
+                                  type: object
+                                headerName:
+                                  type: string
+                                headerOrder:
+                                  type: string
+                                headerValue:
+                                  type: string
+                                name:
+                                  type: string
+                                type:
+                                  enum:
+                                    - apply-content-signatures
+                                    - apply-value-and-content-signatures
+                                    - disallow
+                                    - do-nothing
+                                    - form-data
+                                    - gwt
+                                    - json
+                                    - xml
+                                    - grpc
+                                  type: string
+                              type: object
+                            type: array
+                          wildcardOrder:
+                            type: integer
+                        type: object
+                      type: array
+                    whitelist-ips:
+                      items:
+                        properties:
+                          blockRequests:
+                            enum:
+                              - always
+                              - never
+                              - policy-default
+                            type: string
+                          ipAddress:
+                            pattern: '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'
+                            type: string
+                          ipMask:
+                            pattern: '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}'
+                            type: string
+                        type: object
+                      type: array
+                    whitelistIpReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    xml-profiles:
+                      items:
+                        properties:
+                          attackSignaturesCheck:
+                            type: boolean
+                          defenseAttributes:
+                            properties:
+                              allowCDATA:
+                                type: boolean
+                              allowDTDs:
+                                type: boolean
+                              allowExternalReferences:
+                                type: boolean
+                              allowProcessingInstructions:
+                                type: boolean
+                              maximumAttributeValueLength:
+                                pattern: any|\d+
+                                type: string
+                              maximumAttributesPerElement:
+                                pattern: any|\d+
+                                type: string
+                              maximumChildrenPerElement:
+                                pattern: any|\d+
+                                type: string
+                              maximumDocumentDepth:
+                                pattern: any|\d+
+                                type: string
+                              maximumDocumentSize:
+                                pattern: any|\d+
+                                type: string
+                              maximumElements:
+                                pattern: any|\d+
+                                type: string
+                              maximumNSDeclarations:
+                                pattern: any|\d+
+                                type: string
+                              maximumNameLength:
+                                pattern: any|\d+
+                                type: string
+                              maximumNamespaceLength:
+                                pattern: any|\d+
+                                type: string
+                              tolerateCloseTagShorthand:
+                                type: boolean
+                              tolerateLeadingWhiteSpace:
+                                type: boolean
+                              tolerateNumericNames:
+                                type: boolean
+                            type: object
+                          description:
+                            type: string
+                          enableWss:
+                            type: boolean
+                          followSchemaLinks:
+                            type: boolean
+                          name:
+                            type: string
+                          signatureOverrides:
+                            items:
+                              properties:
+                                enabled:
+                                  type: boolean
+                                name:
+                                  type: string
+                                signatureId:
+                                  type: integer
+                                tag:
+                                  type: string
+                              type: object
+                            type: array
+                        type: object
+                      type: array
+                    xml-validation-files:
+                      items:
+                        properties:
+                          contents:
+                            type: string
+                          fileName:
+                            type: string
+                          isBase64:
+                            type: boolean
+                        type: object
+                      type: array
+                    xmlProfileReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                    xmlValidationFileReference:
+                      properties:
+                        link:
+                          pattern: ^http
+                          type: string
+                      type: object
+                  type: object
+              type: object
+          type: object
+      served: true
+      storage: true
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml
new file mode 100644
index 0000000000..044eeb4563
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/appprotect.f5.com_apusersigs.yaml
@@ -0,0 +1,93 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+  creationTimestamp: null
+  name: apusersigs.appprotect.f5.com
+spec:
+  group: appprotect.f5.com
+  names:
+    kind: APUserSig
+    listKind: APUserSigList
+    plural: apusersigs
+    singular: apusersig
+  preserveUnknownFields: false
+  scope: Namespaced
+  versions:
+    - name: v1beta1
+      schema:
+        openAPIV3Schema:
+          description: APUserSig is the Schema for the apusersigs API
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: APUserSigSpec defines the desired state of APUserSig
+              properties:
+                properties:
+                  type: string
+                signatures:
+                  items:
+                    properties:
+                      accuracy:
+                        enum:
+                          - high
+                          - medium
+                          - low
+                        type: string
+                      attackType:
+                        properties:
+                          name:
+                            type: string
+                        type: object
+                      description:
+                        type: string
+                      name:
+                        type: string
+                      references:
+                        properties:
+                          type:
+                            enum:
+                              - bugtraq
+                              - cve
+                              - nessus
+                              - url
+                            type: string
+                          value:
+                            type: string
+                        type: object
+                      risk:
+                        enum:
+                          - high
+                          - medium
+                          - low
+                        type: string
+                      rule:
+                        type: string
+                      signatureType:
+                        enum:
+                          - request
+                          - response
+                        type: string
+                      systems:
+                        items:
+                          properties:
+                            name:
+                              type: string
+                          type: object
+                        type: array
+                    type: object
+                  type: array
+                tag:
+                  type: string
+              type: object
+          type: object
+      served: true
+      storage: true
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml
new file mode 100644
index 0000000000..3f1a248b1f
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_globalconfigurations.yaml
@@ -0,0 +1,56 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: globalconfigurations.k8s.nginx.org
+spec:
+  group: k8s.nginx.org
+  names:
+    kind: GlobalConfiguration
+    listKind: GlobalConfigurationList
+    plural: globalconfigurations
+    shortNames:
+      - gc
+    singular: globalconfiguration
+  scope: Namespaced
+  versions:
+    - name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: GlobalConfiguration defines the GlobalConfiguration resource.
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: GlobalConfigurationSpec is the spec of the GlobalConfiguration resource.
+              type: object
+              properties:
+                listeners:
+                  type: array
+                  items:
+                    description: Listener defines a listener.
+                    type: object
+                    properties:
+                      name:
+                        type: string
+                      port:
+                        type: integer
+                      protocol:
+                        type: string
+      served: true
+      storage: true
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml
new file mode 100644
index 0000000000..ec583f5448
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_policies.yaml
@@ -0,0 +1,176 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: policies.k8s.nginx.org
+spec:
+  group: k8s.nginx.org
+  names:
+    kind: Policy
+    listKind: PolicyList
+    plural: policies
+    shortNames:
+      - pol
+    singular: policy
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Current state of the Policy. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller.
+          jsonPath: .status.state
+          name: State
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: Policy defines a Policy for VirtualServer and VirtualServerRoute resources.
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: PolicySpec is the spec of the Policy resource. The spec includes multiple fields, where each field represents a different policy. Only one policy (field) is allowed.
+              type: object
+              properties:
+                accessControl:
+                  description: 'AccessControl defines an access policy based on the source IP of a request. policy status: production-ready'
+                  type: object
+                  properties:
+                    allow:
+                      type: array
+                      items:
+                        type: string
+                    deny:
+                      type: array
+                      items:
+                        type: string
+                egressMTLS:
+                  description: 'EgressMTLS defines an Egress MTLS policy. policy status: preview'
+                  type: object
+                  properties:
+                    ciphers:
+                      type: string
+                    protocols:
+                      type: string
+                    serverName:
+                      type: boolean
+                    sessionReuse:
+                      type: boolean
+                    sslName:
+                      type: string
+                    tlsSecret:
+                      type: string
+                    trustedCertSecret:
+                      type: string
+                    verifyDepth:
+                      type: integer
+                    verifyServer:
+                      type: boolean
+                ingressMTLS:
+                  description: 'IngressMTLS defines an Ingress MTLS policy. policy status: preview'
+                  type: object
+                  properties:
+                    clientCertSecret:
+                      type: string
+                    verifyClient:
+                      type: string
+                    verifyDepth:
+                      type: integer
+                jwt:
+                  description: 'JWTAuth holds JWT authentication configuration. policy status: preview'
+                  type: object
+                  properties:
+                    realm:
+                      type: string
+                    secret:
+                      type: string
+                    token:
+                      type: string
+                oidc:
+                  description: OIDC defines an Open ID Connect policy.
+                  type: object
+                  properties:
+                    authEndpoint:
+                      type: string
+                    clientID:
+                      type: string
+                    clientSecret:
+                      type: string
+                    jwksURI:
+                      type: string
+                    redirectURI:
+                      type: string
+                    scope:
+                      type: string
+                    tokenEndpoint:
+                      type: string
+                rateLimit:
+                  description: 'RateLimit defines a rate limit policy. policy status: preview'
+                  type: object
+                  properties:
+                    burst:
+                      type: integer
+                    delay:
+                      type: integer
+                    dryRun:
+                      type: boolean
+                    key:
+                      type: string
+                    logLevel:
+                      type: string
+                    noDelay:
+                      type: boolean
+                    rate:
+                      type: string
+                    rejectCode:
+                      type: integer
+                    zoneSize:
+                      type: string
+                waf:
+                  description: 'WAF defines an WAF policy. policy status: preview'
+                  type: object
+                  properties:
+                    apPolicy:
+                      type: string
+                    enable:
+                      type: boolean
+                    securityLog:
+                      description: SecurityLog defines the security log of a WAF policy.
+                      type: object
+                      properties:
+                        apLogConf:
+                          type: string
+                        enable:
+                          type: boolean
+                        logDest:
+                          type: string
+            status:
+              description: PolicyStatus is the status of the policy resource
+              type: object
+              properties:
+                message:
+                  type: string
+                reason:
+                  type: string
+                state:
+                  type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml
new file mode 100644
index 0000000000..4a28dd9f12
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_transportservers.yaml
@@ -0,0 +1,157 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: transportservers.k8s.nginx.org
+spec:
+  group: k8s.nginx.org
+  names:
+    kind: TransportServer
+    listKind: TransportServerList
+    plural: transportservers
+    shortNames:
+      - ts
+    singular: transportserver
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Current state of the TransportServer. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller.
+          jsonPath: .status.state
+          name: State
+          type: string
+        - jsonPath: .status.reason
+          name: Reason
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1alpha1
+      schema:
+        openAPIV3Schema:
+          description: TransportServer defines the TransportServer resource.
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: TransportServerSpec is the spec of the TransportServer resource.
+              type: object
+              properties:
+                action:
+                  description: Action defines an action.
+                  type: object
+                  properties:
+                    pass:
+                      type: string
+                host:
+                  type: string
+                ingressClassName:
+                  type: string
+                listener:
+                  description: TransportServerListener defines a listener for a TransportServer.
+                  type: object
+                  properties:
+                    name:
+                      type: string
+                    protocol:
+                      type: string
+                serverSnippets:
+                  type: string
+                sessionParameters:
+                  description: SessionParameters defines session parameters.
+                  type: object
+                  properties:
+                    timeout:
+                      type: string
+                streamSnippets:
+                  type: string
+                upstreamParameters:
+                  description: UpstreamParameters defines parameters for an upstream.
+                  type: object
+                  properties:
+                    connectTimeout:
+                      type: string
+                    nextUpstream:
+                      type: boolean
+                    nextUpstreamTimeout:
+                      type: string
+                    nextUpstreamTries:
+                      type: integer
+                    udpRequests:
+                      type: integer
+                    udpResponses:
+                      type: integer
+                upstreams:
+                  type: array
+                  items:
+                    description: Upstream defines an upstream.
+                    type: object
+                    properties:
+                      failTimeout:
+                        type: string
+                      healthCheck:
+                        description: HealthCheck defines the parameters for active Upstream HealthChecks.
+                        type: object
+                        properties:
+                          enable:
+                            type: boolean
+                          fails:
+                            type: integer
+                          interval:
+                            type: string
+                          jitter:
+                            type: string
+                          match:
+                            description: Match defines the parameters of a custom health check.
+                            type: object
+                            properties:
+                              expect:
+                                type: string
+                              send:
+                                type: string
+                          passes:
+                            type: integer
+                          port:
+                            type: integer
+                          timeout:
+                            type: string
+                      loadBalancingMethod:
+                        type: string
+                      maxConns:
+                        type: integer
+                      maxFails:
+                        type: integer
+                      name:
+                        type: string
+                      port:
+                        type: integer
+                      service:
+                        type: string
+            status:
+              description: TransportServerStatus defines the status for the TransportServer resource.
+              type: object
+              properties:
+                message:
+                  type: string
+                reason:
+                  type: string
+                state:
+                  type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml
new file mode 100644
index 0000000000..5d2816cf70
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualserverroutes.yaml
@@ -0,0 +1,621 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: virtualserverroutes.k8s.nginx.org
+spec:
+  group: k8s.nginx.org
+  names:
+    kind: VirtualServerRoute
+    listKind: VirtualServerRouteList
+    plural: virtualserverroutes
+    shortNames:
+      - vsr
+    singular: virtualserverroute
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Current state of the VirtualServerRoute. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller.
+          jsonPath: .status.state
+          name: State
+          type: string
+        - jsonPath: .spec.host
+          name: Host
+          type: string
+        - jsonPath: .status.externalEndpoints[*].ip
+          name: IP
+          type: string
+        - jsonPath: .status.externalEndpoints[*].ports
+          name: Ports
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VirtualServerRoute defines the VirtualServerRoute resource.
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: VirtualServerRouteSpec is the spec of the VirtualServerRoute resource.
+              type: object
+              properties:
+                host:
+                  type: string
+                ingressClassName:
+                  type: string
+                subroutes:
+                  type: array
+                  items:
+                    description: Route defines a route.
+                    type: object
+                    properties:
+                      action:
+                        description: Action defines an action.
+                        type: object
+                        properties:
+                          pass:
+                            type: string
+                          proxy:
+                            description: ActionProxy defines a proxy in an Action.
+                            type: object
+                            properties:
+                              requestHeaders:
+                                description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                type: object
+                                properties:
+                                  pass:
+                                    type: boolean
+                                  set:
+                                    type: array
+                                    items:
+                                      description: Header defines an HTTP Header.
+                                      type: object
+                                      properties:
+                                        name:
+                                          type: string
+                                        value:
+                                          type: string
+                              responseHeaders:
+                                description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                type: object
+                                properties:
+                                  add:
+                                    type: array
+                                    items:
+                                      description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                      type: object
+                                      properties:
+                                        always:
+                                          type: boolean
+                                        name:
+                                          type: string
+                                        value:
+                                          type: string
+                                  hide:
+                                    type: array
+                                    items:
+                                      type: string
+                                  ignore:
+                                    type: array
+                                    items:
+                                      type: string
+                                  pass:
+                                    type: array
+                                    items:
+                                      type: string
+                              rewritePath:
+                                type: string
+                              upstream:
+                                type: string
+                          redirect:
+                            description: ActionRedirect defines a redirect in an Action.
+                            type: object
+                            properties:
+                              code:
+                                type: integer
+                              url:
+                                type: string
+                          return:
+                            description: ActionReturn defines a return in an Action.
+                            type: object
+                            properties:
+                              body:
+                                type: string
+                              code:
+                                type: integer
+                              type:
+                                type: string
+                      errorPages:
+                        type: array
+                        items:
+                          description: ErrorPage defines an ErrorPage in a Route.
+                          type: object
+                          properties:
+                            codes:
+                              type: array
+                              items:
+                                type: integer
+                            redirect:
+                              description: ErrorPageRedirect defines a redirect for an ErrorPage.
+                              type: object
+                              properties:
+                                code:
+                                  type: integer
+                                url:
+                                  type: string
+                            return:
+                              description: ErrorPageReturn defines a return for an ErrorPage.
+                              type: object
+                              properties:
+                                body:
+                                  type: string
+                                code:
+                                  type: integer
+                                headers:
+                                  type: array
+                                  items:
+                                    description: Header defines an HTTP Header.
+                                    type: object
+                                    properties:
+                                      name:
+                                        type: string
+                                      value:
+                                        type: string
+                                type:
+                                  type: string
+                      location-snippets:
+                        type: string
+                      matches:
+                        type: array
+                        items:
+                          description: Match defines a match.
+                          type: object
+                          properties:
+                            action:
+                              description: Action defines an action.
+                              type: object
+                              properties:
+                                pass:
+                                  type: string
+                                proxy:
+                                  description: ActionProxy defines a proxy in an Action.
+                                  type: object
+                                  properties:
+                                    requestHeaders:
+                                      description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        pass:
+                                          type: boolean
+                                        set:
+                                          type: array
+                                          items:
+                                            description: Header defines an HTTP Header.
+                                            type: object
+                                            properties:
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                    responseHeaders:
+                                      description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        add:
+                                          type: array
+                                          items:
+                                            description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                            type: object
+                                            properties:
+                                              always:
+                                                type: boolean
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                        hide:
+                                          type: array
+                                          items:
+                                            type: string
+                                        ignore:
+                                          type: array
+                                          items:
+                                            type: string
+                                        pass:
+                                          type: array
+                                          items:
+                                            type: string
+                                    rewritePath:
+                                      type: string
+                                    upstream:
+                                      type: string
+                                redirect:
+                                  description: ActionRedirect defines a redirect in an Action.
+                                  type: object
+                                  properties:
+                                    code:
+                                      type: integer
+                                    url:
+                                      type: string
+                                return:
+                                  description: ActionReturn defines a return in an Action.
+                                  type: object
+                                  properties:
+                                    body:
+                                      type: string
+                                    code:
+                                      type: integer
+                                    type:
+                                      type: string
+                            conditions:
+                              type: array
+                              items:
+                                description: Condition defines a condition in a MatchRule.
+                                type: object
+                                properties:
+                                  argument:
+                                    type: string
+                                  cookie:
+                                    type: string
+                                  header:
+                                    type: string
+                                  value:
+                                    type: string
+                                  variable:
+                                    type: string
+                            splits:
+                              type: array
+                              items:
+                                description: Split defines a split.
+                                type: object
+                                properties:
+                                  action:
+                                    description: Action defines an action.
+                                    type: object
+                                    properties:
+                                      pass:
+                                        type: string
+                                      proxy:
+                                        description: ActionProxy defines a proxy in an Action.
+                                        type: object
+                                        properties:
+                                          requestHeaders:
+                                            description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                            type: object
+                                            properties:
+                                              pass:
+                                                type: boolean
+                                              set:
+                                                type: array
+                                                items:
+                                                  description: Header defines an HTTP Header.
+                                                  type: object
+                                                  properties:
+                                                    name:
+                                                      type: string
+                                                    value:
+                                                      type: string
+                                          responseHeaders:
+                                            description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                            type: object
+                                            properties:
+                                              add:
+                                                type: array
+                                                items:
+                                                  description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                                  type: object
+                                                  properties:
+                                                    always:
+                                                      type: boolean
+                                                    name:
+                                                      type: string
+                                                    value:
+                                                      type: string
+                                              hide:
+                                                type: array
+                                                items:
+                                                  type: string
+                                              ignore:
+                                                type: array
+                                                items:
+                                                  type: string
+                                              pass:
+                                                type: array
+                                                items:
+                                                  type: string
+                                          rewritePath:
+                                            type: string
+                                          upstream:
+                                            type: string
+                                      redirect:
+                                        description: ActionRedirect defines a redirect in an Action.
+                                        type: object
+                                        properties:
+                                          code:
+                                            type: integer
+                                          url:
+                                            type: string
+                                      return:
+                                        description: ActionReturn defines a return in an Action.
+                                        type: object
+                                        properties:
+                                          body:
+                                            type: string
+                                          code:
+                                            type: integer
+                                          type:
+                                            type: string
+                                  weight:
+                                    type: integer
+                      path:
+                        type: string
+                      policies:
+                        type: array
+                        items:
+                          description: PolicyReference references a policy by name and an optional namespace.
+                          type: object
+                          properties:
+                            name:
+                              type: string
+                            namespace:
+                              type: string
+                      route:
+                        type: string
+                      splits:
+                        type: array
+                        items:
+                          description: Split defines a split.
+                          type: object
+                          properties:
+                            action:
+                              description: Action defines an action.
+                              type: object
+                              properties:
+                                pass:
+                                  type: string
+                                proxy:
+                                  description: ActionProxy defines a proxy in an Action.
+                                  type: object
+                                  properties:
+                                    requestHeaders:
+                                      description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        pass:
+                                          type: boolean
+                                        set:
+                                          type: array
+                                          items:
+                                            description: Header defines an HTTP Header.
+                                            type: object
+                                            properties:
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                    responseHeaders:
+                                      description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        add:
+                                          type: array
+                                          items:
+                                            description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                            type: object
+                                            properties:
+                                              always:
+                                                type: boolean
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                        hide:
+                                          type: array
+                                          items:
+                                            type: string
+                                        ignore:
+                                          type: array
+                                          items:
+                                            type: string
+                                        pass:
+                                          type: array
+                                          items:
+                                            type: string
+                                    rewritePath:
+                                      type: string
+                                    upstream:
+                                      type: string
+                                redirect:
+                                  description: ActionRedirect defines a redirect in an Action.
+                                  type: object
+                                  properties:
+                                    code:
+                                      type: integer
+                                    url:
+                                      type: string
+                                return:
+                                  description: ActionReturn defines a return in an Action.
+                                  type: object
+                                  properties:
+                                    body:
+                                      type: string
+                                    code:
+                                      type: integer
+                                    type:
+                                      type: string
+                            weight:
+                              type: integer
+                upstreams:
+                  type: array
+                  items:
+                    description: Upstream defines an upstream.
+                    type: object
+                    properties:
+                      buffer-size:
+                        type: string
+                      buffering:
+                        type: boolean
+                      buffers:
+                        description: UpstreamBuffers defines Buffer Configuration for an Upstream.
+                        type: object
+                        properties:
+                          number:
+                            type: integer
+                          size:
+                            type: string
+                      client-max-body-size:
+                        type: string
+                      connect-timeout:
+                        type: string
+                      fail-timeout:
+                        type: string
+                      healthCheck:
+                        description: HealthCheck defines the parameters for active Upstream HealthChecks.
+                        type: object
+                        properties:
+                          connect-timeout:
+                            type: string
+                          enable:
+                            type: boolean
+                          fails:
+                            type: integer
+                          headers:
+                            type: array
+                            items:
+                              description: Header defines an HTTP Header.
+                              type: object
+                              properties:
+                                name:
+                                  type: string
+                                value:
+                                  type: string
+                          interval:
+                            type: string
+                          jitter:
+                            type: string
+                          passes:
+                            type: integer
+                          path:
+                            type: string
+                          port:
+                            type: integer
+                          read-timeout:
+                            type: string
+                          send-timeout:
+                            type: string
+                          statusMatch:
+                            type: string
+                          tls:
+                            description: UpstreamTLS defines a TLS configuration for an Upstream.
+                            type: object
+                            properties:
+                              enable:
+                                type: boolean
+                      keepalive:
+                        type: integer
+                      lb-method:
+                        type: string
+                      max-conns:
+                        type: integer
+                      max-fails:
+                        type: integer
+                      name:
+                        type: string
+                      next-upstream:
+                        type: string
+                      next-upstream-timeout:
+                        type: string
+                      next-upstream-tries:
+                        type: integer
+                      port:
+                        type: integer
+                      queue:
+                        description: UpstreamQueue defines Queue Configuration for an Upstream.
+                        type: object
+                        properties:
+                          size:
+                            type: integer
+                          timeout:
+                            type: string
+                      read-timeout:
+                        type: string
+                      send-timeout:
+                        type: string
+                      service:
+                        type: string
+                      sessionCookie:
+                        description: SessionCookie defines the parameters for session persistence.
+                        type: object
+                        properties:
+                          domain:
+                            type: string
+                          enable:
+                            type: boolean
+                          expires:
+                            type: string
+                          httpOnly:
+                            type: boolean
+                          name:
+                            type: string
+                          path:
+                            type: string
+                          secure:
+                            type: boolean
+                      slow-start:
+                        type: string
+                      subselector:
+                        type: object
+                        additionalProperties:
+                          type: string
+                      tls:
+                        description: UpstreamTLS defines a TLS configuration for an Upstream.
+                        type: object
+                        properties:
+                          enable:
+                            type: boolean
+                      use-cluster-ip:
+                        type: boolean
+            status:
+              description: VirtualServerRouteStatus defines the status for the VirtualServerRoute resource.
+              type: object
+              properties:
+                externalEndpoints:
+                  type: array
+                  items:
+                    description: ExternalEndpoint defines the IP and ports used to connect to this resource.
+                    type: object
+                    properties:
+                      ip:
+                        type: string
+                      ports:
+                        type: string
+                message:
+                  type: string
+                reason:
+                  type: string
+                referencedBy:
+                  type: string
+                state:
+                  type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml
new file mode 100644
index 0000000000..ad2c173611
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/crds/k8s.nginx.org_virtualservers.yaml
@@ -0,0 +1,649 @@
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.6.0
+  creationTimestamp: null
+  name: virtualservers.k8s.nginx.org
+spec:
+  group: k8s.nginx.org
+  names:
+    kind: VirtualServer
+    listKind: VirtualServerList
+    plural: virtualservers
+    shortNames:
+      - vs
+    singular: virtualserver
+  scope: Namespaced
+  versions:
+    - additionalPrinterColumns:
+        - description: Current state of the VirtualServer. If the resource has a valid status, it means it has been validated and accepted by the Ingress Controller.
+          jsonPath: .status.state
+          name: State
+          type: string
+        - jsonPath: .spec.host
+          name: Host
+          type: string
+        - jsonPath: .status.externalEndpoints[*].ip
+          name: IP
+          type: string
+        - jsonPath: .status.externalEndpoints[*].ports
+          name: Ports
+          type: string
+        - jsonPath: .metadata.creationTimestamp
+          name: Age
+          type: date
+      name: v1
+      schema:
+        openAPIV3Schema:
+          description: VirtualServer defines the VirtualServer resource.
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: VirtualServerSpec is the spec of the VirtualServer resource.
+              type: object
+              properties:
+                host:
+                  type: string
+                http-snippets:
+                  type: string
+                ingressClassName:
+                  type: string
+                policies:
+                  type: array
+                  items:
+                    description: PolicyReference references a policy by name and an optional namespace.
+                    type: object
+                    properties:
+                      name:
+                        type: string
+                      namespace:
+                        type: string
+                routes:
+                  type: array
+                  items:
+                    description: Route defines a route.
+                    type: object
+                    properties:
+                      action:
+                        description: Action defines an action.
+                        type: object
+                        properties:
+                          pass:
+                            type: string
+                          proxy:
+                            description: ActionProxy defines a proxy in an Action.
+                            type: object
+                            properties:
+                              requestHeaders:
+                                description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                type: object
+                                properties:
+                                  pass:
+                                    type: boolean
+                                  set:
+                                    type: array
+                                    items:
+                                      description: Header defines an HTTP Header.
+                                      type: object
+                                      properties:
+                                        name:
+                                          type: string
+                                        value:
+                                          type: string
+                              responseHeaders:
+                                description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                type: object
+                                properties:
+                                  add:
+                                    type: array
+                                    items:
+                                      description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                      type: object
+                                      properties:
+                                        always:
+                                          type: boolean
+                                        name:
+                                          type: string
+                                        value:
+                                          type: string
+                                  hide:
+                                    type: array
+                                    items:
+                                      type: string
+                                  ignore:
+                                    type: array
+                                    items:
+                                      type: string
+                                  pass:
+                                    type: array
+                                    items:
+                                      type: string
+                              rewritePath:
+                                type: string
+                              upstream:
+                                type: string
+                          redirect:
+                            description: ActionRedirect defines a redirect in an Action.
+                            type: object
+                            properties:
+                              code:
+                                type: integer
+                              url:
+                                type: string
+                          return:
+                            description: ActionReturn defines a return in an Action.
+                            type: object
+                            properties:
+                              body:
+                                type: string
+                              code:
+                                type: integer
+                              type:
+                                type: string
+                      errorPages:
+                        type: array
+                        items:
+                          description: ErrorPage defines an ErrorPage in a Route.
+                          type: object
+                          properties:
+                            codes:
+                              type: array
+                              items:
+                                type: integer
+                            redirect:
+                              description: ErrorPageRedirect defines a redirect for an ErrorPage.
+                              type: object
+                              properties:
+                                code:
+                                  type: integer
+                                url:
+                                  type: string
+                            return:
+                              description: ErrorPageReturn defines a return for an ErrorPage.
+                              type: object
+                              properties:
+                                body:
+                                  type: string
+                                code:
+                                  type: integer
+                                headers:
+                                  type: array
+                                  items:
+                                    description: Header defines an HTTP Header.
+                                    type: object
+                                    properties:
+                                      name:
+                                        type: string
+                                      value:
+                                        type: string
+                                type:
+                                  type: string
+                      location-snippets:
+                        type: string
+                      matches:
+                        type: array
+                        items:
+                          description: Match defines a match.
+                          type: object
+                          properties:
+                            action:
+                              description: Action defines an action.
+                              type: object
+                              properties:
+                                pass:
+                                  type: string
+                                proxy:
+                                  description: ActionProxy defines a proxy in an Action.
+                                  type: object
+                                  properties:
+                                    requestHeaders:
+                                      description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        pass:
+                                          type: boolean
+                                        set:
+                                          type: array
+                                          items:
+                                            description: Header defines an HTTP Header.
+                                            type: object
+                                            properties:
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                    responseHeaders:
+                                      description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        add:
+                                          type: array
+                                          items:
+                                            description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                            type: object
+                                            properties:
+                                              always:
+                                                type: boolean
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                        hide:
+                                          type: array
+                                          items:
+                                            type: string
+                                        ignore:
+                                          type: array
+                                          items:
+                                            type: string
+                                        pass:
+                                          type: array
+                                          items:
+                                            type: string
+                                    rewritePath:
+                                      type: string
+                                    upstream:
+                                      type: string
+                                redirect:
+                                  description: ActionRedirect defines a redirect in an Action.
+                                  type: object
+                                  properties:
+                                    code:
+                                      type: integer
+                                    url:
+                                      type: string
+                                return:
+                                  description: ActionReturn defines a return in an Action.
+                                  type: object
+                                  properties:
+                                    body:
+                                      type: string
+                                    code:
+                                      type: integer
+                                    type:
+                                      type: string
+                            conditions:
+                              type: array
+                              items:
+                                description: Condition defines a condition in a MatchRule.
+                                type: object
+                                properties:
+                                  argument:
+                                    type: string
+                                  cookie:
+                                    type: string
+                                  header:
+                                    type: string
+                                  value:
+                                    type: string
+                                  variable:
+                                    type: string
+                            splits:
+                              type: array
+                              items:
+                                description: Split defines a split.
+                                type: object
+                                properties:
+                                  action:
+                                    description: Action defines an action.
+                                    type: object
+                                    properties:
+                                      pass:
+                                        type: string
+                                      proxy:
+                                        description: ActionProxy defines a proxy in an Action.
+                                        type: object
+                                        properties:
+                                          requestHeaders:
+                                            description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                            type: object
+                                            properties:
+                                              pass:
+                                                type: boolean
+                                              set:
+                                                type: array
+                                                items:
+                                                  description: Header defines an HTTP Header.
+                                                  type: object
+                                                  properties:
+                                                    name:
+                                                      type: string
+                                                    value:
+                                                      type: string
+                                          responseHeaders:
+                                            description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                            type: object
+                                            properties:
+                                              add:
+                                                type: array
+                                                items:
+                                                  description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                                  type: object
+                                                  properties:
+                                                    always:
+                                                      type: boolean
+                                                    name:
+                                                      type: string
+                                                    value:
+                                                      type: string
+                                              hide:
+                                                type: array
+                                                items:
+                                                  type: string
+                                              ignore:
+                                                type: array
+                                                items:
+                                                  type: string
+                                              pass:
+                                                type: array
+                                                items:
+                                                  type: string
+                                          rewritePath:
+                                            type: string
+                                          upstream:
+                                            type: string
+                                      redirect:
+                                        description: ActionRedirect defines a redirect in an Action.
+                                        type: object
+                                        properties:
+                                          code:
+                                            type: integer
+                                          url:
+                                            type: string
+                                      return:
+                                        description: ActionReturn defines a return in an Action.
+                                        type: object
+                                        properties:
+                                          body:
+                                            type: string
+                                          code:
+                                            type: integer
+                                          type:
+                                            type: string
+                                  weight:
+                                    type: integer
+                      path:
+                        type: string
+                      policies:
+                        type: array
+                        items:
+                          description: PolicyReference references a policy by name and an optional namespace.
+                          type: object
+                          properties:
+                            name:
+                              type: string
+                            namespace:
+                              type: string
+                      route:
+                        type: string
+                      splits:
+                        type: array
+                        items:
+                          description: Split defines a split.
+                          type: object
+                          properties:
+                            action:
+                              description: Action defines an action.
+                              type: object
+                              properties:
+                                pass:
+                                  type: string
+                                proxy:
+                                  description: ActionProxy defines a proxy in an Action.
+                                  type: object
+                                  properties:
+                                    requestHeaders:
+                                      description: ProxyRequestHeaders defines the request headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        pass:
+                                          type: boolean
+                                        set:
+                                          type: array
+                                          items:
+                                            description: Header defines an HTTP Header.
+                                            type: object
+                                            properties:
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                    responseHeaders:
+                                      description: ProxyResponseHeaders defines the response headers manipulation in an ActionProxy.
+                                      type: object
+                                      properties:
+                                        add:
+                                          type: array
+                                          items:
+                                            description: AddHeader defines an HTTP Header with an optional Always field to use with the add_header NGINX directive.
+                                            type: object
+                                            properties:
+                                              always:
+                                                type: boolean
+                                              name:
+                                                type: string
+                                              value:
+                                                type: string
+                                        hide:
+                                          type: array
+                                          items:
+                                            type: string
+                                        ignore:
+                                          type: array
+                                          items:
+                                            type: string
+                                        pass:
+                                          type: array
+                                          items:
+                                            type: string
+                                    rewritePath:
+                                      type: string
+                                    upstream:
+                                      type: string
+                                redirect:
+                                  description: ActionRedirect defines a redirect in an Action.
+                                  type: object
+                                  properties:
+                                    code:
+                                      type: integer
+                                    url:
+                                      type: string
+                                return:
+                                  description: ActionReturn defines a return in an Action.
+                                  type: object
+                                  properties:
+                                    body:
+                                      type: string
+                                    code:
+                                      type: integer
+                                    type:
+                                      type: string
+                            weight:
+                              type: integer
+                server-snippets:
+                  type: string
+                tls:
+                  description: TLS defines TLS configuration for a VirtualServer.
+                  type: object
+                  properties:
+                    redirect:
+                      description: TLSRedirect defines a redirect for a TLS.
+                      type: object
+                      properties:
+                        basedOn:
+                          type: string
+                        code:
+                          type: integer
+                        enable:
+                          type: boolean
+                    secret:
+                      type: string
+                upstreams:
+                  type: array
+                  items:
+                    description: Upstream defines an upstream.
+                    type: object
+                    properties:
+                      buffer-size:
+                        type: string
+                      buffering:
+                        type: boolean
+                      buffers:
+                        description: UpstreamBuffers defines Buffer Configuration for an Upstream.
+                        type: object
+                        properties:
+                          number:
+                            type: integer
+                          size:
+                            type: string
+                      client-max-body-size:
+                        type: string
+                      connect-timeout:
+                        type: string
+                      fail-timeout:
+                        type: string
+                      healthCheck:
+                        description: HealthCheck defines the parameters for active Upstream HealthChecks.
+                        type: object
+                        properties:
+                          connect-timeout:
+                            type: string
+                          enable:
+                            type: boolean
+                          fails:
+                            type: integer
+                          headers:
+                            type: array
+                            items:
+                              description: Header defines an HTTP Header.
+                              type: object
+                              properties:
+                                name:
+                                  type: string
+                                value:
+                                  type: string
+                          interval:
+                            type: string
+                          jitter:
+                            type: string
+                          passes:
+                            type: integer
+                          path:
+                            type: string
+                          port:
+                            type: integer
+                          read-timeout:
+                            type: string
+                          send-timeout:
+                            type: string
+                          statusMatch:
+                            type: string
+                          tls:
+                            description: UpstreamTLS defines a TLS configuration for an Upstream.
+                            type: object
+                            properties:
+                              enable:
+                                type: boolean
+                      keepalive:
+                        type: integer
+                      lb-method:
+                        type: string
+                      max-conns:
+                        type: integer
+                      max-fails:
+                        type: integer
+                      name:
+                        type: string
+                      next-upstream:
+                        type: string
+                      next-upstream-timeout:
+                        type: string
+                      next-upstream-tries:
+                        type: integer
+                      port:
+                        type: integer
+                      queue:
+                        description: UpstreamQueue defines Queue Configuration for an Upstream.
+                        type: object
+                        properties:
+                          size:
+                            type: integer
+                          timeout:
+                            type: string
+                      read-timeout:
+                        type: string
+                      send-timeout:
+                        type: string
+                      service:
+                        type: string
+                      sessionCookie:
+                        description: SessionCookie defines the parameters for session persistence.
+                        type: object
+                        properties:
+                          domain:
+                            type: string
+                          enable:
+                            type: boolean
+                          expires:
+                            type: string
+                          httpOnly:
+                            type: boolean
+                          name:
+                            type: string
+                          path:
+                            type: string
+                          secure:
+                            type: boolean
+                      slow-start:
+                        type: string
+                      subselector:
+                        type: object
+                        additionalProperties:
+                          type: string
+                      tls:
+                        description: UpstreamTLS defines a TLS configuration for an Upstream.
+                        type: object
+                        properties:
+                          enable:
+                            type: boolean
+                      use-cluster-ip:
+                        type: boolean
+            status:
+              description: VirtualServerStatus defines the status for the VirtualServer resource.
+              type: object
+              properties:
+                externalEndpoints:
+                  type: array
+                  items:
+                    description: ExternalEndpoint defines the IP and ports used to connect to this resource.
+                    type: object
+                    properties:
+                      ip:
+                        type: string
+                      ports:
+                        type: string
+                message:
+                  type: string
+                reason:
+                  type: string
+                state:
+                  type: string
+      served: true
+      storage: true
+      subresources:
+        status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt b/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt
new file mode 100644
index 0000000000..c61fbd1a23
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/NOTES.txt
@@ -0,0 +1 @@
+The NGINX Ingress Controller has been installed.
\ No newline at end of file
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl b/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl
new file mode 100644
index 0000000000..9a73a5656c
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/_helpers.tpl
@@ -0,0 +1,71 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "nginx-ingress.name" -}}
+{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create labels
+*/}}
+{{- define "nginx-ingress.labels" -}}
+app.kubernetes.io/name: {{ include "nginx-ingress.name" . }}
+helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Expand the name of the configmap.
+*/}}
+{{- define "nginx-ingress.configName" -}}
+{{- default (include "nginx-ingress.name" .) .Values.controller.config.name -}}
+{{- end -}}
+
+{{/*
+Expand leader election lock name.
+*/}}
+{{- define "nginx-ingress.leaderElectionName" -}}
+{{- if .Values.controller.reportIngressStatus.leaderElectionLockName -}}
+{{ .Values.controller.reportIngressStatus.leaderElectionLockName }}
+{{- else -}}
+{{- printf "%s-%s" (include "nginx-ingress.name" .) "leader-election" -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Expand service account name.
+*/}}
+{{- define "nginx-ingress.serviceAccountName" -}}
+{{- default (include "nginx-ingress.name" .) .Values.controller.serviceAccount.name -}}
+{{- end -}}
+
+{{/*
+Expand service name.
+*/}}
+{{- define "nginx-ingress.serviceName" -}}
+{{- default (include "nginx-ingress.name" .) .Values.controller.service.name }}
+{{- end -}}
+
+{{/*
+Expand default TLS name.
+*/}}
+{{- define "nginx-ingress.defaultTLSName" -}}
+{{- printf "%s-%s" (include "nginx-ingress.name" .) "default-server-tls" -}}
+{{- end -}}
+
+{{/*
+Expand wildcard TLS name.
+*/}}
+{{- define "nginx-ingress.wildcardTLSName" -}}
+{{- printf "%s-%s" (include "nginx-ingress.name" .) "wildcard-tls" -}}
+{{- end -}}
+
+{{/*
+Expand app name.
+*/}}
+{{- define "nginx-ingress.appName" -}}
+{{- default (include "nginx-ingress.name" .) .Values.controller.name -}}
+{{- end -}}
\ No newline at end of file
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml
new file mode 100644
index 0000000000..971faa84d0
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-configmap.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nginx-ingress.configName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+{{- if .Values.controller.config.annotations }}
+  annotations:
+{{ toYaml .Values.controller.config.annotations | indent 4 }}
+{{- end }}
+data:
+{{- if .Values.controller.config.entries }}
+{{ toYaml .Values.controller.config.entries | indent 2 }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml
new file mode 100644
index 0000000000..e39ab74534
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-daemonset.yaml
@@ -0,0 +1,159 @@
+{{- if eq .Values.controller.kind "daemonset" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: {{ default (include "nginx-ingress.name" .) .Values.controller.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: {{ include "nginx-ingress.appName" . }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "nginx-ingress.appName" . }}
+{{- if or (.Values.prometheus.create) (.Values.controller.pod.annotations) }}
+      annotations:
+{{- if .Values.prometheus.create }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.prometheus.port }}"
+        prometheus.io/scheme: "{{ .Values.prometheus.scheme }}"
+{{- end }}
+{{- if .Values.controller.pod.annotations }}
+{{ toYaml .Values.controller.pod.annotations | indent 8 }}
+{{- end }}
+{{- end }}
+    spec:
+      serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }}
+      terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }}
+{{- if .Values.controller.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.controller.nodeSelector | indent 8 }}
+{{- end }}
+{{- if .Values.controller.tolerations }}
+      tolerations:
+{{ toYaml .Values.controller.tolerations | indent 6 }}
+{{- end }}
+{{- if .Values.controller.affinity }}
+      affinity:
+{{ toYaml .Values.controller.affinity | indent 8 }}
+{{- end }}
+{{- if .Values.controller.volumes }}
+      volumes:
+{{ toYaml .Values.controller.volumes | indent 6 }}
+{{- end }}
+{{- if .Values.controller.priorityClassName }}
+      priorityClassName: {{ .Values.controller.priorityClassName }}
+{{- end }}
+      hostNetwork: {{ .Values.controller.hostNetwork }}
+      containers:
+      - name: {{ include "nginx-ingress.name" . }}
+        image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}"
+        imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}"
+        ports:
+        - name: http
+          containerPort: 80
+          hostPort: 80
+        - name: https
+          containerPort: 443
+          hostPort: 443
+{{ if .Values.controller.customPorts }}
+{{ toYaml .Values.controller.customPorts | indent 8 }}
+{{ end }}
+{{- if .Values.prometheus.create }}
+        - name: prometheus
+          containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
+{{- end }}
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
+{{- if .Values.controller.volumeMounts }}
+        volumeMounts:
+{{ toYaml .Values.controller.volumeMounts | indent 8 }}
+{{- end }}
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        resources:
+{{ toYaml .Values.controller.resources | indent 10 }}
+        args:
+          - -nginx-plus={{ .Values.controller.nginxplus }}
+          - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }}
+          - -enable-app-protect={{ .Values.controller.appprotect.enable }}
+          - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }}
+{{- if .Values.controller.defaultTLS.secret }}
+          - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }}
+{{ else }}
+          - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }}
+{{- end }}
+          - -ingress-class={{ .Values.controller.ingressClass }}
+{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }}
+          - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }}
+{{- end }}
+{{- if .Values.controller.watchNamespace }}
+          - -watch-namespace={{ .Values.controller.watchNamespace }}
+{{- end }}
+          - -health-status={{ .Values.controller.healthStatus }}
+          - -health-status-uri={{ .Values.controller.healthStatusURI }}
+          - -nginx-debug={{ .Values.controller.nginxDebug }}
+          - -v={{ .Values.controller.logLevel }}
+          - -nginx-status={{ .Values.controller.nginxStatus.enable }}
+{{- if .Values.controller.nginxStatus.enable }}
+          - -nginx-status-port={{ .Values.controller.nginxStatus.port }}
+          - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }}
+{{- end }}
+{{- if .Values.controller.reportIngressStatus.enable }}
+          - -report-ingress-status
+{{- if .Values.controller.reportIngressStatus.ingressLink }}
+          - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }}
+{{- else if .Values.controller.reportIngressStatus.externalService }}
+          - -external-service={{ .Values.controller.reportIngressStatus.externalService }}
+{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }}
+          - -external-service={{ include "nginx-ingress.serviceName" . }}
+{{- end }}
+          - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }}
+          - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }}
+{{- end }}
+{{- if .Values.controller.wildcardTLS.secret }}
+          - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }}
+{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }}
+          - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }}
+{{- end }}
+          - -enable-prometheus-metrics={{ .Values.prometheus.create }}
+          - -prometheus-metrics-listen-port={{ .Values.prometheus.port }}
+          - -prometheus-tls-secret={{ .Values.prometheus.secret }}
+          - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
+{{- if .Values.controller.enableCustomResources }}
+          - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
+          - -enable-snippets={{ .Values.controller.enableSnippets }}
+          - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }}
+{{- if .Values.controller.globalConfiguration.create }}
+          - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.name" . }}
+{{- end }}
+{{- end }}
+          - -ready-status={{ .Values.controller.readyStatus.enable }}
+          - -ready-status-port={{ .Values.controller.readyStatus.port }}
+          - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml
new file mode 100644
index 0000000000..4b6cd91746
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-deployment.yaml
@@ -0,0 +1,157 @@
+{{- if eq .Values.controller.kind "deployment" }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ default (include "nginx-ingress.name" .) .Values.controller.name }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+spec:
+  replicas: {{ .Values.controller.replicaCount }}
+  selector:
+    matchLabels:
+      app: {{ include "nginx-ingress.appName" . }}
+  template:
+    metadata:
+      labels:
+        app: {{ include "nginx-ingress.appName" . }}
+{{- if or (.Values.prometheus.create) (.Values.controller.pod.annotations) }}
+      annotations:
+{{- if .Values.prometheus.create }}
+        prometheus.io/scrape: "true"
+        prometheus.io/port: "{{ .Values.prometheus.port }}"
+        prometheus.io/scheme: "{{ .Values.prometheus.scheme }}"
+{{- end }}
+{{- if .Values.controller.pod.annotations }}
+{{ toYaml .Values.controller.pod.annotations | indent 8 }}
+{{- end }}
+{{- end }}
+    spec:
+{{- if .Values.controller.nodeSelector }}
+      nodeSelector:
+{{ toYaml .Values.controller.nodeSelector | indent 8 }}
+{{- end }}
+{{- if .Values.controller.tolerations }}
+      tolerations:
+{{ toYaml .Values.controller.tolerations | indent 6 }}
+{{- end }}
+{{- if .Values.controller.affinity }}
+      affinity:
+{{ toYaml .Values.controller.affinity | indent 8 }}
+{{- end }}
+{{- if .Values.controller.volumes }}
+      volumes:
+{{ toYaml .Values.controller.volumes | indent 6 }}
+{{- end }}
+{{- if .Values.controller.priorityClassName }}
+      priorityClassName: {{ .Values.controller.priorityClassName }}
+{{- end }}
+      serviceAccountName: {{ include "nginx-ingress.serviceAccountName" . }}
+      hostNetwork: {{ .Values.controller.hostNetwork }}
+      containers:
+      - image: "{{ .Values.controller.image.repository }}:{{ .Values.controller.image.tag }}"
+        name: {{ include "nginx-ingress.name" . }}
+        imagePullPolicy: "{{ .Values.controller.image.pullPolicy }}"
+        ports:
+        - name: http
+          containerPort: 80
+        - name: https
+          containerPort: 443
+{{ if .Values.controller.customPorts }}
+{{ toYaml .Values.controller.customPorts | indent 8 }}
+{{ end }}
+{{- if .Values.prometheus.create }}
+        - name: prometheus
+          containerPort: {{ .Values.prometheus.port }}
+{{- end }}
+{{- if .Values.controller.readyStatus.enable }}
+        - name: readiness-port
+          containerPort: {{ .Values.controller.readyStatus.port}}
+        readinessProbe:
+          httpGet:
+            path: /nginx-ready
+            port: readiness-port
+          periodSeconds: 1
+{{- end }}
+        resources:
+{{ toYaml .Values.controller.resources | indent 10 }}
+        securityContext:
+          allowPrivilegeEscalation: true
+          runAsUser: 101 #nginx
+          capabilities:
+            drop:
+            - ALL
+            add:
+            - NET_BIND_SERVICE
+{{- if .Values.controller.volumeMounts }}
+        volumeMounts:
+{{ toYaml .Values.controller.volumeMounts | indent 8 }}
+{{- end }}
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        args:
+          - -nginx-plus={{ .Values.controller.nginxplus }}
+          - -nginx-reload-timeout={{ .Values.controller.nginxReloadTimeout }}
+          - -enable-app-protect={{ .Values.controller.appprotect.enable }}
+          - -nginx-configmaps=$(POD_NAMESPACE)/{{ include "nginx-ingress.configName" . }}
+{{- if .Values.controller.defaultTLS.secret }}
+          - -default-server-tls-secret={{ .Values.controller.defaultTLS.secret }}
+{{ else }}
+          - -default-server-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.defaultTLSName" . }}
+{{- end }}
+          - -ingress-class={{ .Values.controller.ingressClass }}
+{{- if semverCompare "<1.18.0" .Capabilities.KubeVersion.GitVersion }}
+          - -use-ingress-class-only={{ .Values.controller.useIngressClassOnly }}
+{{- end }}
+{{- if .Values.controller.watchNamespace }}
+          - -watch-namespace={{ .Values.controller.watchNamespace }}
+{{- end }}
+          - -health-status={{ .Values.controller.healthStatus }}
+          - -health-status-uri={{ .Values.controller.healthStatusURI }}
+          - -nginx-debug={{ .Values.controller.nginxDebug }}
+          - -v={{ .Values.controller.logLevel }}
+          - -nginx-status={{ .Values.controller.nginxStatus.enable }}
+{{- if .Values.controller.nginxStatus.enable }}
+          - -nginx-status-port={{ .Values.controller.nginxStatus.port }}
+          - -nginx-status-allow-cidrs={{ .Values.controller.nginxStatus.allowCidrs }}
+{{- end }}
+{{- if .Values.controller.reportIngressStatus.enable }}
+          - -report-ingress-status
+{{- if .Values.controller.reportIngressStatus.ingressLink }}
+          - -ingresslink={{ .Values.controller.reportIngressStatus.ingressLink }}
+{{- else if .Values.controller.reportIngressStatus.externalService }}
+          - -external-service={{ .Values.controller.reportIngressStatus.externalService }}
+{{- else if and (.Values.controller.service.create) (eq .Values.controller.service.type "LoadBalancer") }}
+          - -external-service={{ include "nginx-ingress.serviceName" . }}
+{{- end }}
+          - -enable-leader-election={{ .Values.controller.reportIngressStatus.enableLeaderElection }}
+          - -leader-election-lock-name={{ include "nginx-ingress.leaderElectionName" . }}
+{{- end }}
+{{- if .Values.controller.wildcardTLS.secret }}
+          - -wildcard-tls-secret={{ .Values.controller.wildcardTLS.secret }}
+{{- else if and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key }}
+          - -wildcard-tls-secret=$(POD_NAMESPACE)/{{ include "nginx-ingress.wildcardTLSName" . }}
+{{- end }}
+          - -enable-prometheus-metrics={{ .Values.prometheus.create }}
+          - -prometheus-metrics-listen-port={{ .Values.prometheus.port }}
+          - -prometheus-tls-secret={{ .Values.prometheus.secret }}
+          - -enable-custom-resources={{ .Values.controller.enableCustomResources }}
+{{- if .Values.controller.enableCustomResources }}
+          - -enable-tls-passthrough={{ .Values.controller.enableTLSPassthrough }}
+          - -enable-snippets={{ .Values.controller.enableSnippets }}
+          - -enable-preview-policies={{ .Values.controller.enablePreviewPolicies }}
+{{- if .Values.controller.globalConfiguration.create }}
+          - -global-configuration=$(POD_NAMESPACE)/{{ include "nginx-ingress.name" . }}
+{{- end }}
+{{- end }}
+          - -ready-status={{ .Values.controller.readyStatus.enable }}
+          - -ready-status-port={{ .Values.controller.readyStatus.port }}
+          - -enable-latency-metrics={{ .Values.controller.enableLatencyMetrics }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml
new file mode 100644
index 0000000000..b0bba48704
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-globalconfiguration.yaml
@@ -0,0 +1,11 @@
+{{ if .Values.controller.globalConfiguration.create }}
+apiVersion: k8s.nginx.org/v1alpha1
+kind: GlobalConfiguration
+metadata:
+  name: {{ include "nginx-ingress.name" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+spec:
+{{ toYaml .Values.controller.globalConfiguration.spec | indent 2 }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml
new file mode 100644
index 0000000000..04df66a30f
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-ingress-class.yaml
@@ -0,0 +1,12 @@
+{{- if semverCompare ">=1.18.0-0" .Capabilities.KubeVersion.GitVersion }}
+apiVersion: networking.k8s.io/v1beta1
+kind: IngressClass
+metadata:
+  name: {{ .Values.controller.ingressClass }}
+{{- if .Values.controller.setAsDefaultIngress }}
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+{{- end }}
+spec:
+  controller: nginx.org/ingress-controller
+{{- end }}
\ No newline at end of file
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml
new file mode 100644
index 0000000000..ef6f79a581
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-leader-election-configmap.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "nginx-ingress.leaderElectionName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+{{- if .Values.controller.reportIngressStatus.annotations }}
+  annotations:
+{{ toYaml .Values.controller.reportIngressStatus.annotations | indent 4 }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml
new file mode 100644
index 0000000000..4c9e693bb5
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-secret.yaml
@@ -0,0 +1,13 @@
+{{ if not .Values.controller.defaultTLS.secret }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "nginx-ingress.defaultTLSName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ .Values.controller.defaultTLS.cert }}
+  tls.key: {{ .Values.controller.defaultTLS.key }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml
new file mode 100644
index 0000000000..11d542271f
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-service.yaml
@@ -0,0 +1,60 @@
+{{- if .Values.controller.service.create }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "nginx-ingress.serviceName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+{{- if .Values.controller.service.extraLabels }}
+{{ toYaml .Values.controller.service.extraLabels | indent 4 }}
+{{- end }}
+{{- if .Values.controller.service.annotations }}
+  annotations:
+{{ toYaml .Values.controller.service.annotations | indent 4 }}
+{{- end }}
+spec:
+{{- if or (eq .Values.controller.service.type "LoadBalancer") (eq .Values.controller.service.type "NodePort") }}
+  {{- if .Values.controller.service.externalTrafficPolicy }}
+  externalTrafficPolicy: {{ .Values.controller.service.externalTrafficPolicy }}
+  {{- end }}
+{{- end }}
+{{- if eq .Values.controller.service.type "LoadBalancer" }}
+  {{- if .Values.controller.service.loadBalancerIP }}
+  loadBalancerIP: {{ .Values.controller.service.loadBalancerIP }}
+  {{- end }}
+  {{- if .Values.controller.service.loadBalancerSourceRanges }}
+  loadBalancerSourceRanges:
+{{ toYaml .Values.controller.service.loadBalancerSourceRanges | indent 4 }}
+  {{- end }}
+{{- end }}
+  type: {{ .Values.controller.service.type }}
+  ports:
+{{- if .Values.controller.service.customPorts }}
+{{ toYaml .Values.controller.service.customPorts | indent 2 }}
+{{ end }}
+{{- if .Values.controller.service.httpPort.enable }}
+  - port: {{ .Values.controller.service.httpPort.port }}
+    targetPort: {{ .Values.controller.service.httpPort.targetPort }}
+    protocol: TCP
+    name: http
+  {{- if eq .Values.controller.service.type "NodePort" }}
+    nodePort: {{ .Values.controller.service.httpPort.nodePort }}
+  {{- end }}
+{{- end }}
+{{- if .Values.controller.service.httpsPort.enable }}
+  - port: {{ .Values.controller.service.httpsPort.port }}
+    targetPort: {{ .Values.controller.service.httpsPort.targetPort }}
+    protocol: TCP
+    name: https
+  {{- if eq .Values.controller.service.type "NodePort" }}
+    nodePort: {{ .Values.controller.service.httpsPort.nodePort }}
+  {{- end }}
+{{- end }}
+  selector:
+    app:  {{ include "nginx-ingress.appName" . }}
+  {{- if .Values.controller.service.externalIPs }}
+  externalIPs:
+{{ toYaml .Values.controller.service.externalIPs | indent 4 }}
+  {{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml
new file mode 100644
index 0000000000..c03c6a793b
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-serviceaccount.yaml
@@ -0,0 +1,13 @@
+{{- if .Values.rbac.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "nginx-ingress.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+{{- if .Values.controller.serviceAccount.imagePullSecretName }}
+imagePullSecrets:
+- name: {{ .Values.controller.serviceAccount.imagePullSecretName }}
+{{- end }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml b/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml
new file mode 100644
index 0000000000..3abe16c4c3
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/controller-wildcard-secret.yaml
@@ -0,0 +1,13 @@
+{{ if and (not .Values.controller.wildcardTLS.secret) (and .Values.controller.wildcardTLS.cert .Values.controller.wildcardTLS.key) }}
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ include "nginx-ingress.wildcardTLSName" . }}
+  namespace: {{ .Release.Namespace }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+type: kubernetes.io/tls
+data:
+  tls.crt: {{ .Values.controller.wildcardTLS.cert }}
+  tls.key: {{ .Values.controller.wildcardTLS.key }}
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml b/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml
new file mode 100644
index 0000000000..7538b4830d
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/templates/rbac.yaml
@@ -0,0 +1,135 @@
+{{- if .Values.rbac.create }}
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "nginx-ingress.name" . }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+rules:
+{{- if .Values.controller.appprotect.enable }}
+- apiGroups: 
+  - appprotect.f5.com
+  resources: 
+  - appolicies
+  - aplogconfs
+  - apusersigs
+  verbs: 
+  - get 
+  - watch
+  - list
+{{- end }}
+- apiGroups:
+  - ""
+  resources:
+  - services
+  - endpoints
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+{{- if .Values.controller.reportIngressStatus.enableLeaderElection }}
+  - update
+  - create
+{{- end }}
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - list
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingressclasses
+  verbs:
+  - get
+{{- if .Values.controller.reportIngressStatus.enable }}
+- apiGroups:
+  - networking.k8s.io
+  resources:
+  - ingresses/status
+  verbs:
+  - update
+{{- end }}
+{{- if .Values.controller.enableCustomResources }}
+- apiGroups:
+  - k8s.nginx.org
+  resources:
+  - virtualservers
+  - virtualserverroutes
+  - globalconfigurations
+  - transportservers
+  - policies
+  verbs:
+  - list
+  - watch
+  - get
+- apiGroups:
+  - k8s.nginx.org
+  resources:
+  - virtualservers/status
+  - virtualserverroutes/status
+  - policies/status
+  - transportservers/status
+  verbs:
+  - update
+{{- end }}
+{{- if .Values.controller.reportIngressStatus.ingressLink }}
+- apiGroups:
+  - cis.f5.com
+  resources:
+  - ingresslinks
+  verbs:
+  - list
+  - watch
+  - get
+{{- end }}
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: {{ include "nginx-ingress.name" . }}
+  labels:
+    {{- include "nginx-ingress.labels" . | nindent 4 }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "nginx-ingress.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "nginx-ingress.name" . }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/kubernetes/helm_charts/nginx-ingress/values-icp.yaml b/kubernetes/helm_charts/nginx-ingress/values-icp.yaml
new file mode 100644
index 0000000000..6b74235fa1
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/values-icp.yaml
@@ -0,0 +1,16 @@
+controller:
+  kind: daemonset
+  nginxplus: true
+  image:
+    repository: mycluster.icp:8500/kube-system/nginx-plus-ingress
+    tag: "1.12.0"
+  nodeSelector:
+    beta.kubernetes.io/arch: "amd64"
+    proxy: true
+  terminationGracePeriodSeconds: 60
+  tolerations:
+    - key: "dedicated"
+      operator: "Exists"
+      effect: "NoSchedule"
+    - key: "CriticalAddonsOnly"
+      operator: "Exists"
diff --git a/kubernetes/helm_charts/nginx-ingress/values-plus.yaml b/kubernetes/helm_charts/nginx-ingress/values-plus.yaml
new file mode 100644
index 0000000000..24ab2844e2
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/values-plus.yaml
@@ -0,0 +1,5 @@
+controller:
+  nginxplus: true
+  image:
+    repository: nginx-plus-ingress
+    tag: "1.12.0"
diff --git a/kubernetes/helm_charts/nginx-ingress/values.yaml b/kubernetes/helm_charts/nginx-ingress/values.yaml
new file mode 100644
index 0000000000..ec9e782397
--- /dev/null
+++ b/kubernetes/helm_charts/nginx-ingress/values.yaml
@@ -0,0 +1,319 @@
+controller:
+  ## The name of the Ingress controller daemonset or deployment.
+  ## Autogenerated if not set or set to "".
+  # name: nginx-ingress
+
+  ## The kind of the Ingress controller installation - deployment or daemonset.
+  kind: deployment
+
+  ## Deploys the Ingress controller for NGINX Plus.
+  nginxplus: false
+
+  # Timeout in milliseconds which the Ingress Controller will wait for a successful NGINX reload after a change or at the initial start.
+  # Default is 4000 (default is 20000 instead if enable-app-protect is true)
+  # If set to 0, default values will be used.
+  nginxReloadTimeout: 0
+
+  ## Support for App Protect
+  appprotect:
+    ## Enable the App Protect module in the Ingress Controller.
+    enable: false
+
+  ## Enables the Ingress controller pods to use the host's network namespace.
+  hostNetwork: false
+
+  ## Enables debugging for NGINX. Uses the nginx-debug binary. Requires error-log-level: debug in the ConfigMap via `controller.config.entries`.
+  nginxDebug: false
+
+  ## The log level of the Ingress Controller.
+  logLevel: 1
+
+  ## A list of custom ports to expose on the NGINX ingress controller pod. Follows the conventional Kubernetes yaml syntax for container ports.
+  customPorts: []
+
+  image:
+    ## The image repository of the Ingress controller.
+    repository: nginx/nginx-ingress
+
+    ## The tag of the Ingress controller image.
+    tag: "1.12.0"
+
+    ## The pull policy for the Ingress controller image.
+    pullPolicy: IfNotPresent
+
+  config:
+    ## The name of the ConfigMap used by the Ingress controller.
+    ## Autogenerated if not set or set to "".
+    # name: nginx-config
+
+    ## The annotations of the Ingress Controller configmap.
+    annotations: {}
+
+    ## The entries of the ConfigMap for customizing NGINX configuration.
+    entries: {}
+
+  ## It is recommended to use your own TLS certificates and keys
+  defaultTLS:
+    ## The base64-encoded TLS certificate for the default HTTPS server. If not specified, a pre-generated self-signed certificate is used.
+    ## Note: It is recommended that you specify your own certificate.
+    cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN2akNDQWFZQ0NRREFPRjl0THNhWFhEQU5CZ2txaGtpRzl3MEJBUXNGQURBaE1SOHdIUVlEVlFRRERCWk8KUjBsT1dFbHVaM0psYzNORGIyNTBjbTlzYkdWeU1CNFhEVEU0TURreE1qRTRNRE16TlZvWERUSXpNRGt4TVRFNApNRE16TlZvd0lURWZNQjBHQTFVRUF3d1dUa2RKVGxoSmJtZHlaWE56UTI5dWRISnZiR3hsY2pDQ0FTSXdEUVlKCktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQUwvN2hIUEtFWGRMdjNyaUM3QlBrMTNpWkt5eTlyQ08KR2xZUXYyK2EzUDF0azIrS3YwVGF5aGRCbDRrcnNUcTZzZm8vWUk1Y2Vhbkw4WGM3U1pyQkVRYm9EN2REbWs1Qgo4eDZLS2xHWU5IWlg0Rm5UZ0VPaStlM2ptTFFxRlBSY1kzVnNPazFFeUZBL0JnWlJVbkNHZUtGeERSN0tQdGhyCmtqSXVuektURXUyaDU4Tlp0S21ScUJHdDEwcTNRYzhZT3ExM2FnbmovUWRjc0ZYYTJnMjB1K1lYZDdoZ3krZksKWk4vVUkxQUQ0YzZyM1lma1ZWUmVHd1lxQVp1WXN2V0RKbW1GNWRwdEMzN011cDBPRUxVTExSakZJOTZXNXIwSAo1TmdPc25NWFJNV1hYVlpiNWRxT3R0SmRtS3FhZ25TZ1JQQVpQN2MwQjFQU2FqYzZjNGZRVXpNQ0F3RUFBVEFOCkJna3Foa2lHOXcwQkFRc0ZBQU9DQVFFQWpLb2tRdGRPcEsrTzhibWVPc3lySmdJSXJycVFVY2ZOUitjb0hZVUoKdGhrYnhITFMzR3VBTWI5dm15VExPY2xxeC9aYzJPblEwMEJCLzlTb0swcitFZ1U2UlVrRWtWcitTTFA3NTdUWgozZWI4dmdPdEduMS9ienM3bzNBaS9kclkrcUI5Q2k1S3lPc3FHTG1US2xFaUtOYkcyR1ZyTWxjS0ZYQU80YTY3Cklnc1hzYktNbTQwV1U3cG9mcGltU1ZmaXFSdkV5YmN3N0NYODF6cFErUyt1eHRYK2VBZ3V0NHh3VlI5d2IyVXYKelhuZk9HbWhWNThDd1dIQnNKa0kxNXhaa2VUWXdSN0diaEFMSkZUUkk3dkhvQXprTWIzbjAxQjQyWjNrN3RXNQpJUDFmTlpIOFUvOWxiUHNoT21FRFZkdjF5ZytVRVJxbStGSis2R0oxeFJGcGZnPT0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=
+
+    ## The base64-encoded TLS key for the default HTTPS server. Note: If not specified, a pre-generated key is used.
+    ## Note: It is recommended that you specify your own key.
+    key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFcEFJQkFBS0NBUUVBdi91RWM4b1JkMHUvZXVJTHNFK1RYZUprckxMMnNJNGFWaEMvYjVyYy9XMlRiNHEvClJOcktGMEdYaVN1eE9ycXgrajlnamx4NXFjdnhkenRKbXNFUkJ1Z1B0ME9hVGtIekhvb3FVWmcwZGxmZ1dkT0EKUTZMNTdlT1l0Q29VOUZ4amRXdzZUVVRJVUQ4R0JsRlNjSVo0b1hFTkhzbysyR3VTTWk2Zk1wTVM3YUhudzFtMApxWkdvRWEzWFNyZEJ6eGc2clhkcUNlUDlCMXl3VmRyYURiUzc1aGQzdUdETDU4cGszOVFqVUFQaHpxdmRoK1JWClZGNGJCaW9CbTVpeTlZTW1hWVhsMm0wTGZzeTZuUTRRdFFzdEdNVWozcGJtdlFmazJBNnljeGRFeFpkZFZsdmwKMm82MjBsMllxcHFDZEtCRThCay90elFIVTlKcU56cHpoOUJUTXdJREFRQUJBb0lCQVFDZklHbXowOHhRVmorNwpLZnZJUXQwQ0YzR2MxNld6eDhVNml4MHg4Mm15d1kxUUNlL3BzWE9LZlRxT1h1SENyUlp5TnUvZ2IvUUQ4bUFOCmxOMjRZTWl0TWRJODg5TEZoTkp3QU5OODJDeTczckM5bzVvUDlkazAvYzRIbjAzSkVYNzZ5QjgzQm9rR1FvYksKMjhMNk0rdHUzUmFqNjd6Vmc2d2szaEhrU0pXSzBwV1YrSjdrUkRWYmhDYUZhNk5nMUZNRWxhTlozVDhhUUtyQgpDUDNDeEFTdjYxWTk5TEI4KzNXWVFIK3NYaTVGM01pYVNBZ1BkQUk3WEh1dXFET1lvMU5PL0JoSGt1aVg2QnRtCnorNTZud2pZMy8yUytSRmNBc3JMTnIwMDJZZi9oY0IraVlDNzVWYmcydVd6WTY3TWdOTGQ5VW9RU3BDRkYrVm4KM0cyUnhybnhBb0dCQU40U3M0ZVlPU2huMVpQQjdhTUZsY0k2RHR2S2ErTGZTTXFyY2pOZjJlSEpZNnhubmxKdgpGenpGL2RiVWVTbWxSekR0WkdlcXZXaHFISy9iTjIyeWJhOU1WMDlRQ0JFTk5jNmtWajJTVHpUWkJVbEx4QzYrCk93Z0wyZHhKendWelU0VC84ajdHalRUN05BZVpFS2FvRHFyRG5BYWkyaW5oZU1JVWZHRXFGKzJyQW9HQkFOMVAKK0tZL0lsS3RWRzRKSklQNzBjUis3RmpyeXJpY05iWCtQVzUvOXFHaWxnY2grZ3l4b25BWlBpd2NpeDN3QVpGdwpaZC96ZFB2aTBkWEppc1BSZjRMazg5b2pCUmpiRmRmc2l5UmJYbyt3TFU4NUhRU2NGMnN5aUFPaTVBRHdVU0FkCm45YWFweUNweEFkREtERHdObit3ZFhtaTZ0OHRpSFRkK3RoVDhkaVpBb0dCQUt6Wis1bG9OOTBtYlF4VVh5YUwKMjFSUm9tMGJjcndsTmVCaWNFSmlzaEhYa2xpSVVxZ3hSZklNM2hhUVRUcklKZENFaHFsV01aV0xPb2I2NTNyZgo3aFlMSXM1ZUtka3o0aFRVdnpldm9TMHVXcm9CV2xOVHlGanIrSWhKZnZUc0hpOGdsU3FkbXgySkJhZUFVWUNXCndNdlQ4NmNLclNyNkQrZG8wS05FZzFsL0FvR0FlMkFVdHVFbFNqLzBmRzgrV3hHc1RFV1JqclRNUzRSUjhRWXQKeXdjdFA4aDZxTGxKUTRCWGxQU05rMXZLTmtOUkxIb2pZT2pCQTViYjhibXNVU1BlV09NNENoaFJ4QnlHbmR2eAphYkJDRkFwY0IvbEg4d1R0alVZYlN5T294ZGt5OEp0ek90ajJhS0FiZHd6NlArWDZDODhjZmxYVFo5MWpYL3RMCjF3TmRKS2tDZ1lCbyt0UzB5TzJ2SWFmK2UwSkN5TGhzVDQ5cTN3Zis2QWVqWGx2WDJ1VnRYejN5QTZnbXo5aCsKcDNlK2JMRUxwb3B0WFhNdUFRR0xhUkcrYlNNcjR5dERYbE5ZSndUeThXczNKY3dlSTdqZVp2b0ZpbmNvVlVIMwphdmxoTUVCRGYxSjltSDB5cDBwWUNaS2ROdHNvZEZtQktzVEtQMjJhTmtsVVhCS3gyZzR6cFE9PQotLS0tLUVORCBSU0EgUFJJVkFURSBLRVktLS0tLQo=
+
+    ## The secret with a TLS certificate and key for the default HTTPS server.
+    ## The value must follow the following format: `<namespace>/<name>`.
+    ## Used as an alternative to specifying a certificate and key using `controller.defaultTLS.cert` and `controller.defaultTLS.key` parameters.
+    ## Format: <namespace>/<secret_name>
+    secret:
+
+  wildcardTLS:
+    ## The base64-encoded TLS certificate for every Ingress host that has TLS enabled but no secret specified.
+    ## If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection.
+    cert: ""
+
+    ## The base64-encoded TLS key for every Ingress host that has TLS enabled but no secret specified.
+    ## If the parameter is not set, for such Ingress hosts NGINX will break any attempt to establish a TLS connection.
+    key: ""
+
+    ## The secret with a TLS certificate and key for every Ingress host that has TLS enabled but no secret specified.
+    ## The value must follow the following format: `<namespace>/<name>`.
+    ## Used as an alternative to specifying a certificate and key using `controller.wildcardTLS.cert` and `controller.wildcardTLS.key` parameters.
+    ## Format: <namespace>/<secret_name>
+    secret:
+
+  ## The node selector for pod assignment for the Ingress controller pods.
+  nodeSelector: {}
+
+  ## The termination grace period of the Ingress controller pod.
+  terminationGracePeriodSeconds: 30
+
+  ## The resources of the Ingress controller pods.
+  resources: {}
+    # limits:
+    #   cpu: 100m
+    #   memory: 64Mi
+    # requests:
+    #   cpu: 100m
+    #   memory: 64Mi
+
+  ## The tolerations of the Ingress controller pods.
+  tolerations: []
+
+  ## The affinity of the Ingress controller pods.
+  affinity: {}
+
+  ## The volumes of the Ingress controller pods.
+  volumes: []
+  # - name: extra-conf
+  #   configMap:
+  #     name: extra-conf
+
+  ## The volumeMounts of the Ingress controller pods.
+  volumeMounts: []
+  # - name: extra-conf
+  #   mountPath: /etc/nginx/conf.d/extra.conf
+  #   subPath: extra.conf
+
+  ## The number of replicas of the Ingress controller deployment.
+  replicaCount: 1
+
+  ## A class of the Ingress controller.
+
+  ## For Kubernetes >= 1.18, a corresponding IngressClass resource with the name equal to the class must be deployed. Otherwise,
+  ## the Ingress Controller will fail to start.
+  ## The Ingress controller only processes resources that belong to its class - i.e. have the "ingressClassName" field resource equal to the class.
+
+  ## For Kubernetes < 1.18, the Ingress Controller only processes resources that belong to its class -
+  ## i.e have the annotation "kubernetes.io/ingress.class" (for Ingress resources)
+  ## or field "ingressClassName" (for VirtualServer/VirtualServerRoute/TransportServer resources) equal to the class.
+  ## Additionally, the Ingress Controller processes resources that do not have the class set,
+  ## which can be disabled by setting the controller.useIngressClassOnly parameter to true.
+
+  ## The Ingress Controller processes all the VirtualServer/VirtualServerRoute/TransportServer resources that do not have the "ingressClassName" field for all versions of kubernetes.
+  ingressClass: nginx
+
+  ## For kubernetes versions >= 1.18 this flag will be IGNORED.
+  ## Ignore Ingress resources without the "kubernetes.io/ingress.class" annotation
+  useIngressClassOnly: false
+
+  ## Only for Kubernetes >= 1.18
+  ## New Ingresses without an ingressClassName field specified will be assigned the class specified in `controller.ingressClass`.
+  setAsDefaultIngress: false
+
+  ## Namespace to watch for Ingress resources. By default the Ingress controller watches all namespaces.
+  watchNamespace: ""
+
+  ## Enable the custom resources.
+  enableCustomResources: false
+
+  ## Enable preview policies.
+  enablePreviewPolicies: false
+
+  ## Enable TLS Passthrough on port 443. Requires controller.enableCustomResources.
+  enableTLSPassthrough: false
+
+  globalConfiguration:
+    ## Creates the GlobalConfiguration custom resource. Requires controller.enableCustomResources.
+    create: false
+
+    ## The spec of the GlobalConfiguration for defining the global configuration parameters of the Ingress Controller.
+    spec: {}
+      # listeners:
+      # - name: dns-udp
+      #   port: 5353
+      #   protocol: UDP
+      # - name: dns-tcp
+      #   port: 5353
+      #   protocol: TCP
+
+  ## Enable custom NGINX configuration snippets in VirtualServer, VirtualServerRoute and TransportServer resources.
+  enableSnippets: false
+
+  ## Add a location based on the value of health-status-uri to the default server. The location responds with the 200 status code for any request.
+  ## Useful for external health-checking of the Ingress controller.
+  healthStatus: false
+
+  ## Sets the URI of health status location in the default server. Requires controller.healthStatus.
+  healthStatusURI: "/nginx-health"
+
+  nginxStatus:
+    ## Enable the NGINX stub_status, or the NGINX Plus API.
+    enable: true
+
+    ## Set the port where the NGINX stub_status or the NGINX Plus API is exposed.
+    port: 8080
+
+    ## Add IPv4 IP/CIDR blocks to the allow list for NGINX stub_status or the NGINX Plus API. Separate multiple IP/CIDR by commas.
+    allowCidrs: "127.0.0.1"
+
+  service:
+    ## Creates a service to expose the Ingress controller pods.
+    create: true
+
+    ## The type of service to create for the Ingress controller.
+    type: LoadBalancer
+
+    ## The externalTrafficPolicy of the service. The value Local preserves the client source IP.
+    externalTrafficPolicy: Local
+
+    ## The annotations of the Ingress controller service.
+    annotations:
+      service.beta.kubernetes.io/azure-load-balancer-internal: "true"
+
+    ## The extra labels of the service.
+    extraLabels: {}
+
+    ## The static IP address for the load balancer. Requires controller.service.type set to LoadBalancer. The cloud provider must support this feature.
+    loadBalancerIP: ""
+
+    ## The list of external IPs for the Ingress controller service.
+    externalIPs: []
+
+    ## The IP ranges (CIDR) that are allowed to access the load balancer. Requires controller.service.type set to LoadBalancer. The cloud provider must support this feature.
+    loadBalancerSourceRanges: []
+
+    ## The name of the service
+    ## Autogenerated if not set or set to "".
+    # name: nginx-ingress
+
+    httpPort:
+      ## Enables the HTTP port for the Ingress controller service.
+      enable: true
+
+      ## The HTTP port of the Ingress controller service.
+      port: 80
+
+      ## The custom NodePort for the HTTP port. Requires controller.service.type set to NodePort.
+      nodePort: ""
+
+      ## The HTTP port on the POD where the Ingress controller service is running.
+      targetPort: 80
+
+    httpsPort:
+      ## Enables the HTTPS port for the Ingress controller service.
+      enable: true
+
+      ## The HTTPS port of the Ingress controller service.
+      port: 443
+
+      ## The custom NodePort for the HTTPS port. Requires controller.service.type set to NodePort.
+      nodePort: ""
+
+      ## The HTTPS port on the POD where the Ingress controller service is running.
+      targetPort: 443
+
+    ## A list of custom ports to expose through the Ingress controller service. Follows the conventional Kubernetes yaml syntax for service ports.
+    customPorts: []
+
+  serviceAccount:
+    ## The name of the service account of the Ingress controller pods. Used for RBAC.
+    ## Autogenerated if not set or set to "".
+    # name: nginx-ingress
+
+    ## The name of the secret containing docker registry credentials.
+    ## Secret must exist in the same namespace as the helm release.
+    imagePullSecretName: ""
+
+  reportIngressStatus:
+    ## Updates the address field in the status of Ingress resources with an external address of the Ingress controller.
+    ## You must also specify the source of the external address either through an external service via controller.reportIngressStatus.externalService,
+    ## controller.reportIngressStatus.ingressLink or the external-status-address entry in the ConfigMap via controller.config.entries.
+    ## Note: controller.config.entries.external-status-address takes precedence over the others.
+    enable: true
+
+    ## Specifies the name of the service with the type LoadBalancer through which the Ingress controller is exposed externally.
+    ## The external address of the service is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources.
+    ## controller.reportIngressStatus.enable must be set to true.
+    ## The default is autogenerated and matches the created service (see controller.service.create).
+    # externalService: nginx-ingress
+
+    ## Specifies the name of the IngressLink resource, which exposes the Ingress Controller pods via a BIG-IP system.
+    ## The IP of the BIG-IP system is used when reporting the status of Ingress, VirtualServer and VirtualServerRoute resources.
+    ## controller.reportIngressStatus.enable must be set to true.
+    ingressLink: ""
+
+    ## Enable Leader election to avoid multiple replicas of the controller reporting the status of Ingress resources. controller.reportIngressStatus.enable must be set to true.
+    enableLeaderElection: true
+
+    ## Specifies the name of the ConfigMap, within the same namespace as the controller, used as the lock for leader election. controller.reportIngressStatus.enableLeaderElection must be set to true.
+    ## Autogenerated if not set or set to "".
+    # leaderElectionLockName: "nginx-ingress-leader-election"
+
+    ## The annotations of the leader election configmap.
+    annotations: {}
+
+  pod:
+    ## The annotations of the Ingress Controller pod.
+    annotations: {}
+
+  ## The PriorityClass of the ingress controller pods.
+  priorityClassName:
+
+  readyStatus:
+    ## Enables readiness endpoint "/nginx-ready". The endpoint returns a success code when NGINX has loaded all the config after startup.
+    enable: true
+
+    ## Set the port where the readiness endpoint is exposed.
+    port: 8081
+
+  ## Enable collection of latency metrics for upstreams. Requires prometheus.create.
+  enableLatencyMetrics: false
+
+rbac:
+  ## Configures RBAC.
+  create: true
+
+prometheus:
+  ## Expose NGINX or NGINX Plus metrics in the Prometheus format.
+  create: true
+
+  ## Configures the port to scrape the metrics.
+  port: 9113
+
+  ## Specifies the namespace/name of a Kubernetes TLS Secret which will be used to protect the Prometheus endpoint.
+  secret: ""
+
+  ## Configures the HTTP scheme used.
+  scheme: http
diff --git a/kubernetes/helm_charts/secor/.helmignore b/kubernetes/helm_charts/secor/.helmignore
new file mode 100644
index 0000000000..fbe01f88f2
--- /dev/null
+++ b/kubernetes/helm_charts/secor/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file
diff --git a/kubernetes/helm_charts/secor/Chart.yaml b/kubernetes/helm_charts/secor/Chart.yaml
new file mode 100644
index 0000000000..1f85986391
--- /dev/null
+++ b/kubernetes/helm_charts/secor/Chart.yaml
@@ -0,0 +1,17 @@
+name: secor
+version: 0.1.0
+appVersion: 0.25
+description: Secor is a service implementing Kafka log persistence
+keywords:
+- kafka
+- secor
+maintainers:
+- name: secor-maintainers
+sources:
+- https://github.com/pinterest/secor
+engine: gotpl
+home: https://github.com/pinterest/secor
+
+dependencies:
+  - name: alertrules
+    condition: alertrules.enabled
diff --git a/kubernetes/helm_charts/secor/charts/alertrules/.helmignore b/kubernetes/helm_charts/secor/charts/alertrules/.helmignore
new file mode 100644
index 0000000000..50af031725
--- /dev/null
+++ b/kubernetes/helm_charts/secor/charts/alertrules/.helmignore
@@ -0,0 +1,22 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
diff --git a/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml b/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml
new file mode 100644
index 0000000000..85e318cf75
--- /dev/null
+++ b/kubernetes/helm_charts/secor/charts/alertrules/Chart.yaml
@@ -0,0 +1,21 @@
+apiVersion: v2
+name: alertrules
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.16.0
diff --git a/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl b/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl
new file mode 100644
index 0000000000..95896c3e8c
--- /dev/null
+++ b/kubernetes/helm_charts/secor/charts/alertrules/templates/_helpers.tpl
@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "alertrules.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "alertrules.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "alertrules.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "alertrules.labels" -}}
+helm.sh/chart: {{ include "alertrules.chart" . }}
+{{ include "alertrules.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "alertrules.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "alertrules.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "alertrules.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "alertrules.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}
diff --git a/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml b/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml
new file mode 100644
index 0000000000..01395cc960
--- /dev/null
+++ b/kubernetes/helm_charts/secor/charts/alertrules/templates/promrulesSecorLag.yaml
@@ -0,0 +1,66 @@
+{{- if .Values.secor_alertrule.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  labels:
+    role: alert-rules
+    app: {{ .Values.prometheus_rule_selector_app }}
+    release: {{ .Values.prometheus_rule_selector_release }}
+  name: sunbird-monitoring-{{ .Release.Name }}-secor-rules
+  namespace: monitoring
+spec:
+  groups:
+  - name: alertrules.secor_jobs_lag_rule
+    rules:
+    - alert: secor job {{ .Release.Name }} lag 
+      expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="processing-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }}
+      for: 30m
+      labels:
+        severity: critical
+        module: dp_lag
+        alerttype: lag
+      annotations:
+        message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }}
+        lag: {{`({{ humanize $value }})`}}
+        job_id: {{ .Release.Name }}
+        alertname: SecorJobsLag
+
+    - alert: secor job {{ .Release.Name }} lag 
+      expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="processing-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }}
+      for: 30m
+      labels:
+        severity: warning
+        module: dp_lag
+        alerttype: lag
+      annotations:
+        message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }}
+        lag: {{`({{ humanize $value }})`}}
+        job_id: {{ .Release.Name }}
+        alertname: SecorJobsLag
+
+    - alert: secor job {{ .Release.Name }} lag 
+      expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="ingestion-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }}
+      for: 30m
+      labels:
+        severity: critical
+        module: dp_lag
+        alerttype: lag
+      annotations:
+        message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_critical" }}
+        lag: {{`({{ humanize $value }})`}}
+        job_id: {{ .Release.Name }}
+        alertname: SecorJobsLag
+
+    - alert: secor job {{ .Release.Name }} lag 
+      expr: kafka_consumergroup_lag_sum{consumergroup="{{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}", job="ingestion-kafka-exporter"} > {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }}
+      for: 30m
+      labels:
+        severity: warning
+        module: dp_lag
+        alerttype: lag
+      annotations:
+        message: The secor job {{ .Release.Name }} lag is above threshold {{ get (get $.Values.secor_jobs $.Release.Name) "lag_threshold_warning" }}
+        lag: {{`({{ humanize $value }})`}}
+        job_id: {{ .Release.Name }}
+        alertname: SecorJobsLag
+{{- end }}
diff --git a/kubernetes/helm_charts/secor/config/log4j.docker.properties b/kubernetes/helm_charts/secor/config/log4j.docker.properties
new file mode 100644
index 0000000000..06a1176823
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/log4j.docker.properties
@@ -0,0 +1,8 @@
+# log4j logging configuration.
+
+# root logger.
+log4j.rootLogger=INFO, CONSOLE
+
+log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender
+log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout
+log4j.appender.CONSOLE.layout.ConversionPattern=%d{ISO8601} [%t] (%c) %-5p %m%n
diff --git a/kubernetes/helm_charts/secor/config/secor.azure.properties b/kubernetes/helm_charts/secor/config/secor.azure.properties
new file mode 100644
index 0000000000..23f8b76dc9
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/secor.azure.properties
@@ -0,0 +1,27 @@
+include=secor.properties
+
+# Configure upload manager class to use Azure blob storage upload manager
+secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager
+
+############
+# MUST SET #
+############
+
+# Microsoft Azure blob storage default endpoint protocol 
+secor.azure.endpoints.protocol=https
+
+# Microsoft Azure authentication credentials.
+# https://azure.microsoft.com/en-us/documentation/articles/storage-create-storage-account
+secor.azure.account.name={{ $.Values.azure_account }}
+secor.azure.account.key={{ $.Values.azure_secret }}
+
+# Microsoft Azure blob storage container name. Container is a grouping of a set
+# of blobs. https://msdn.microsoft.com/en-us/library/dd135715.aspx
+secor.azure.container.name={{ $.Values.azure_container_name }}
+
+# Microsoft Azure blob storage path where files are stored within the container.
+secor.azure.path={{- get (get $.Values.secor_jobs $.Release.Name) "base_path" }}
+
+################
+# END MUST SET #
+################
\ No newline at end of file
diff --git a/kubernetes/helm_charts/secor/config/secor.common.properties b/kubernetes/helm_charts/secor/config/secor.common.properties
new file mode 100644
index 0000000000..7050ebcf1b
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/secor.common.properties
@@ -0,0 +1,495 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+############
+# MUST SET #
+############
+
+# Regular expression matching names of consumed topics.
+secor.kafka.topic_filter={{ get (get $.Values.secor_jobs $.Release.Name) "topic" }}
+secor.kafka.topic_blacklist=
+
+# Choose what to fill according to the service you are using
+# in the choice option you can fill S3, GS, Swift or Azure
+cloud.service=Azure
+
+# AWS authentication credentials.
+# Leave empty if using IAM role-based authentication with s3a filesystem.
+aws.access.key=
+aws.secret.key=
+aws.role=
+
+# Optional Proxy Setting. Set to true to enable proxy
+# Only applicable to S3UploadManager
+aws.proxy.isEnabled=false
+aws.proxy.http.host=
+aws.proxy.http.port=
+
+################
+# END MUST SET #
+################
+
+
+# AWS region or endpoint. region should be a known region name (eg.
+# us-east-1). endpoint should be a known S3 endpoint url. If neither
+# are specified, then the default region (us-east-1) is used. If both
+# are specified then endpoint is used.
+#
+# Only apply if the the S3UploadManager is used - see
+# secor.upload.manager.class.
+#
+# http://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
+aws.region=
+aws.endpoint=
+
+# Toggle the AWS S3 client between virtual host style access and path style
+# access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html
+aws.client.pathstyleaccess=false
+
+###########################
+# START AWS S3 ENCRYPTION #
+###########################
+
+# AWS specify type of server-side encryption, if any
+# set to S3 to enable S3-managed encryption
+# set to KMS to enable AWS KMS-managed encryption (see aws.sse.kms.key)
+# set to customer to enable customer-managed encryption (see aws.sse.customer.key)
+# set empty to disable encryption
+aws.sse.type=
+
+# Key to use for S3 server-side encryption, base64-encoded
+# Note: requires aws.sse.type to be set to customer to be used
+aws.sse.customer.key=
+
+# KMS Key to use for S3 server-side encryption, base64-encoded
+# Leave empty to use default generated key
+# Note: requires aws.sse.type to be set to  KMS to be used
+aws.sse.kms.key=
+
+#########################
+# END AWS S3 ENCRYPTION #
+#########################
+
+# Hadoop filesystem to use. Choices are s3n or s3a.
+# See https://wiki.apache.org/hadoop/AmazonS3 for details.
+secor.s3.filesystem=s3n
+
+# Swift config, MUST configure if cloud.service=Swift
+
+# Swift Login Details:
+swift.use.get.auth=true
+swift.auth.url=
+swift.tenant=
+swift.username=
+swift.port=8080
+swift.public=true
+
+# only needed if "swift.use.get.auth" = false
+swift.password=
+
+# only needed if "swift.use.get.auth" = true
+swift.api.key=
+
+# GS config, MUST configure if gcloud.service=GS
+
+# Name of the Google cloud storage bucket where log files are stored.
+secor.gs.bucket=secor_gs
+
+# Google cloud storage path where files are stored within the bucket.
+secor.gs.path=data
+
+# Use direct uploads
+# WARNING: disables resumable uploads, files are uploaded in a single request
+# This may help prevent IOException: insufficient data written,
+# see https://github.com/pinterest/secor/issues/177
+# https://cloud.google.com/storage/docs/json_api/v1/how-tos/upload
+secor.gs.upload.direct=false
+
+# Application credentials configuration file
+# https://developers.google.com/identity/protocols/application-default-credentials
+# It can be empty when secor running in Google Cloud VMs with proper scopes
+secor.gs.credentials.path=
+
+# Zookeeper config.
+zookeeper.session.timeout.ms=3000
+zookeeper.sync.time.ms=200
+
+# Zookeeper path (chroot) under which secor data will be placed.
+secor.zookeeper.path=/
+
+# Impacts how frequently the upload logic is triggered if no messages are delivered.
+kafka.consumer.timeout.ms=10000
+
+# Where consumer should read from if no committed offset in zookeeper.
+#   "smallest" -> read from earliest offset
+#   "largest"  -> read from latest offset
+# Always use "smallest" unless you know what you're doing and are willing to risk
+# data loss for new topics or topics whose number of partitions has changed.
+# See the kafka docs for "auto.offset.reset".
+kafka.consumer.auto.offset.reset=smallest
+
+# Choose between range and roundrobin partition assignment strategy for kafka
+# high level consumers. Check PartitionAssignor.scala in kafa 821 module for
+# the differences between the two.
+# In kafka 811, only range strategy is supported.
+kafka.partition.assignment.strategy=range
+
+# Max number of retries during rebalance.
+kafka.rebalance.max.retries=
+
+# Rebalance backoff.
+kafka.rebalance.backoff.ms=
+
+# Kafka consumer receive buffer size (socket.receive.buffer.bytes)
+kafka.socket.receive.buffer.bytes=
+
+# Kafka fetch max size (fetch.message.max.bytes)
+kafka.fetch.message.max.bytes=5242880
+
+# Kafka fetch min bytes (fetch.fetch.min.bytes)
+kafka.fetch.min.bytes=
+
+# Kafka fetch max wait ms (fetch.max.wait.ms)
+kafka.fetch.wait.max.ms=
+
+# Port of the broker serving topic partition metadata.
+#kafka.seed.broker.port=9092
+
+# Zookeeper path at which kafka is registered. In Zookeeper parlance, this is referred
+# to as the chroot.
+kafka.zookeeper.path=/
+
+# Store offset in zookeeper and kafka consumer topic.
+# Only used if kafka.offsets.storage is set to "kafka"
+# http://kafka.apache.org/documentation.html#oldconsumerconfigs
+# Possible values: true or false
+kafka.dual.commit.enabled=true
+
+# Storage offset.
+# Possible values: "zookeeper" to read offset from zookeeper or "kafka" to read offset from kafka consumer topic
+kafka.offsets.storage=zookeeper
+
+# Parameter which tells whether to extract Kafka message timestamp. This value is to be chose in case of 0.10.x kafka brokers.
+# Default value is false. Also specify `kafka.message.timestamp.className` as `com.pinterest.secor.timestamp.Kafka10MessageTimestamp`,
+# in case you are enabling this parameter as `true`.
+kafka.useTimestamp=false
+
+# Classname for the timestamp field you want to use. Default is `com.pinterest.secor.timestamp.Kafka10MessageTimestamp`
+# for 0.10 build profile. Basically, it will be `Kafka8MessageTimestamp` for 0.8 kafka and `Kafka10MessageTimestamp`
+# for 0.10 kafka. Fully classified names are `com.pinterest.secor.timestamp.Kafka8MessageTimestamp` and
+# `com.pinterest.secor.timestamp.Kafka10MessageTimestamp`.
+kafka.message.timestamp.className=com.pinterest.secor.timestamp.Kafka10MessageTimestamp
+
+# Secor generation is a version that should be incremented during non-backwards-compatible
+# Secor releases. Generation number is one of the components of generated log file names.
+# Generation number makes sure that outputs of different Secor versions are isolated.
+secor.generation=1
+
+# Number of consumer threads per Secor process.
+secor.consumer.threads=7
+
+# Consumption rate limit enforced at the process level (not a consumer-thread level).
+secor.messages.per.second=10000
+
+# Used by the "backup" consumer group only.
+# Number of continuous message offsets that constitute a single offset= partition on s3.
+# Example:
+#   if set to 10,
+#     messages with offsets 0 to 9 will be written to s3 path s3n://.../offset=0/...
+#     messages with offsets 10 to 19 will be written to s3 path s3n://.../offset=10/...
+#     ...
+secor.offsets.per.partition=10000000
+secor.offsets.prefix=offset=
+# How long does it take for secor to forget a topic partition. Applies to stats generation only.
+secor.topic_partition.forget.seconds=600
+
+# Setting the partitioner to use hourly partition
+# By default, the partitioner will do daily partition, so the data will be
+# written into
+#       s3n://.../topic/dt=2015-07-07/
+# If this parameter is set to true, the data will be written into
+#       s3n://.../topic/dt=2015-07-07/hr=02
+# The hour folder ranges from 00 to 23
+partitioner.granularity.hour=false
+partitioner.granularity.minute=false
+
+partitioner.granularity.date.prefix=dt=
+partitioner.granularity.hour.prefix=hr=
+partitioner.granularity.minute.prefix=min=
+
+partitioner.granularity.date.format=yyyy-MM-dd
+partitioner.granularity.hour.format=HH
+partitioner.granularity.minute.format=mm
+
+# how many seconds should the finalizer wait to finalize a partition
+partitioner.finalizer.delay.seconds=3600
+
+# During partition finalization, the finalizer will start from the last
+# time partition (e.g. dt=2015-07-17) and traverse backwards for n
+# partition periods (e.g. dt=2015-07-16, dt=2015-07-15 ...)
+# This parameter controls how many partition periods to traverse back
+# The default is 10
+# secor.finalizer.lookback.periods=10
+
+# If greater than 0, upon startup Secor will clean up directories and files under secor.local.path
+# that are older than this value.
+secor.local.log.delete.age.hours=-1
+
+# Secor comes with a tool that adds Hive partitions for finalized topics. Currently, we support
+# only Hive clusters accessible through Qubole. The token gives access to the Qubole API.
+# It is available at https://api.qubole.com/users/edit
+qubole.api.token=
+
+# hive tables are generally named after the topics. For instance if the topic
+# is request_log the hive table is also called request_log. If you want this
+# to be pinlog_request_log you can set this config to "pinlog_". This affects
+# all topics.
+hive.table.prefix=
+
+# You can also name your hive table directly if your hive table doesn't
+# follow the pattern of <hive.table.prefix><kafka topic>
+# E.g.  hive.table.name.topic1=table1 to indicate that hive table for
+# kafka topic <topic1> will be named <table1>
+
+# Secor can export stats such as consumption lag (in seconds and offsets) per topic partition.
+# Leave empty to disable this functionality.
+tsdb.hostport=
+
+# Regex of topics that are not exported to TSDB.
+monitoring.blacklist.topics=
+
+# Prefix of exported stats.
+monitoring.prefix=secor
+
+# Monitoring interval.
+# Set to 0 to disable - the progress monitor will run once and exit.
+monitoring.interval.seconds=0
+
+# Secor can export stats to statsd such as consumption lag (in seconds and offsets) per topic partition.
+# Leave empty to disable this functionality.
+statsd.hostport=
+
+# Thrift protocol class. It applies to timestamp extractor below and parquet output for thrift messages.
+# TBinaryProtocol by default
+secor.thrift.protocol.class=
+
+# Thrift message class. It applies to parquet output.
+# If all Kafka topics transfer the same thrift message type, set secor.thrift.message.class.*=<thrift class name>
+secor.thrift.message.class.*=
+
+# If true, the consumer group will be the initial prefix of all
+# exported metrics, before `monitoring.prefix` (if set).
+#
+# Setting to false and use monitoring.prefix can lead to nice paths.
+# For example,
+#   secor.kafka.group = secor_hr_partition
+#   monitoring.prefix = secor.hr
+#   statsd.prefixWithConsumerGroup = false
+#   => secor.hr.lag.offsets.<topic>.<partition>
+#
+#   secor.kafka.group = secor_hr_partition
+#   monitoring.prefix = secor
+#   statsd.prefixWithConsumerGroup = true
+#   => secor_hr_partition.secor.lag.offsets.<topic>.<partition>
+statsd.prefixWithConsumerGroup=true
+
+# Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123)
+message.timestamp.name={{ get (get $.Values.secor_jobs $.Release.Name) "timestamp_key" }}
+message.fallback.timestamp.name={{ get (get $.Values.secor_jobs $.Release.Name) "fallback_timestamp_key" }}
+
+# Separator for defining message.timestamp.name in a nested structure. E.g.
+# {"meta_data": {"created": "1405911096123", "last_modified": "1405912096123"}, "data": "test"}
+# message.timestamp.name=meta_data.created
+# message.timestamp.name.separator=.
+message.timestamp.name.separator=
+
+# Field ID of the field that contains timestamp for Thrift message parser.
+# N.B. setting this past 1 will come with a performance penalty
+message.timestamp.id=1
+
+# Data type of the timestamp field for thrift message parser.
+# Supports i64 and i32.
+message.timestamp.type=i64
+
+# Name of field that contains a timestamp, as a date Format, for JSON. (2014-08-07, Jul 23 02:16:57 2005, etc...)
+# Should be used when there is no timestamp in a Long format. Also ignore time zones.
+message.timestamp.input.pattern=yyyy-MM-dd
+
+# whether timestamp field is required, it should always be required.  But
+# for historical reason, we didn't enforce this check, there might exist some
+# installations with messages missing timestamp field
+message.timestamp.required=true
+
+# To enable compression, set this to a valid compression codec implementing
+# org.apache.hadoop.io.compress.CompressionCodec interface, such as
+# 'org.apache.hadoop.io.compress.GzipCodec'.
+secor.compression.codec=org.apache.hadoop.io.compress.GzipCodec
+
+# To set a custom file extension set this to a valid file suffix, such as
+# '.gz', '.part', etc.
+secor.file.extension=.gz
+
+# The secor file reader/writer used to read/write the data, by default we write sequence files
+#secor.file.reader.writer.factory=com.pinterest.secor.io.impl.SequenceFileReaderWriterFactory
+secor.file.reader.writer.factory=com.pinterest.secor.io.impl.DelimitedTextFileReaderWriterFactory
+#if left blank defaults to \n
+secor.file.reader.Delimiter=\n
+#if left blank no Delimiter is added. do not use \ as that needs to be escaped and is an escape
+#character and not a delimtier.
+secor.file.writer.Delimiter=\n
+
+# Max message size in bytes to retrieve via KafkaClient. This is used by ProgressMonitor and PartitionFinalizer.
+# This should be set large enough to accept the max message size configured in your kafka broker
+# Default is 0.1 MB
+secor.max.message.size.bytes=100000
+
+# Class that will manage uploads. Default is to use the hadoop
+# interface to S3.
+secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager
+
+#Set below property to your timezone, and the events will be parsed and converted to the timezone specified
+secor.message.timezone=UTC
+
+#Set below property to your timezone, and partitions in s3 will be created as per timezone provided
+secor.parser.timezone=Asia/Kolkata
+
+# Transformer class that transform message accordingly.
+secor.message.transformer.class=com.pinterest.secor.transformer.IdentityMessageTransformer
+
+# Set below property to true if you want to have the md5hash appended to your s3 path.
+# This helps for better partitioning of the data on s3. Which gives better performance while reading and writing on s3
+secor.s3.prefix.md5hash=false
+
+# After the given date, secor will upload files to the supplied s3 alternative path
+secor.s3.alter.path.date=
+
+# An alternative S3 path for secor to upload files to
+secor.s3.alternative.path=
+
+# If enabled, add calls will be made to qubole, otherwise, skip qubole call for finalization
+secor.enable.qubole=true
+
+# Timeout value for qubole calls
+secor.qubole.timeout.ms=300000
+
+# Topics to upload at a fixed minute mark
+secor.kafka.upload_at_minute_mark.topic_filter=
+
+# What the minute mark is. This isn't triggered unless the topic name matches
+secor.upload.minute_mark=0
+
+# File age per topic and per partition is checked against secor.max.file.age.seconds by looking at
+# the youngest file when true or at the oldest file when false.
+secor.file.age.youngest=true
+
+# Class that manages metric collection.
+# Sending metrics to Ostrich is the default implementation.
+secor.monitoring.metrics.collector.class=com.pinterest.secor.monitoring.OstrichMetricCollector
+
+# Row group size in bytes for Parquet writers. Specifies how much data will be buffered in memory before flushing a
+# block to disk. Larger values allow for larger column chinks which makes it possible to do larger sequential IO.
+# Should be aligned with HDFS blocks. Defaults to 128MB in Parquet 1.9.
+parquet.block.size=134217728
+
+# Page group size in bytes for Parquet writers. Indivisible unit for columnar data. Smaller data pages allow for more
+# fine grained reading but have higher space overhead. Defaults to 1MB in Parquet 1.9.
+parquet.page.size=1048576
+
+# Enable or disable dictionary encoding for Parquet writers. The dictionary encoding builds a dictionary of values
+# encountered in a given column. Defaults to true in Parquet 1.9.
+parquet.enable.dictionary=true
+
+# Enable or disable validation for Parquet writers. Validates records written against the schema. Defaults to false in
+# Parquet 1.9.
+parquet.validation=false
+
+# User can configure ORC schema for each Kafka topic. Common schema is also possible. This property is mandatory
+# if DefaultORCSchemaProvider is used. ORC schema for all the topics should be defined like this:
+secor.orc.message.schema.*=struct<a:int\,b:int\,c:struct<d:int\,e:string>\,f:array<string>\,g:int>
+# Below config used for defining ORC schema provider class name. User can use the custom implementation for orc schema provider
+secor.orc.schema.provider=com.pinterest.secor.util.orc.schema.DefaultORCSchemaProvider
+
+# Changes related to 0.29 upgrade
+
+# If true, runs a final upload when the JVM is shut down (eg, from SIGTERM).
+# You may want to ensure that whatever system you use to run secor sets an
+# appropriate grace period to allow a full upload before a forced termination.
+secor.upload.on.shutdown=false
+
+# If true, uploads lastSeen offset onto ZK, see https://github.com/pinterest/secor/issues/600 for details
+secor.upload.secor.upload.last.seen.offset=false
+
+# If true, uploads are entirely deterministic, which can avoid some race conditions
+# which can lead to messages being backed up multiple times. This is incompatible with
+# secor.upload.on.shutdown=true, and ignores the values of secor.max.file.size.bytes,
+# secor.max.file.age.seconds, and secor.upload.minute_mark.
+#
+# In deterministic mode, you must set one or both of secor.max.file.timestamp.range.millis and
+# secor.max.input.payload.size.bytes. These determine when to upload, and are ignored outside
+# of deterministic mode.
+secor.upload.deterministic=false
+
+# Classname for the message iterator you want to use. The message iterator determines what kind of consumer
+# secor will use to communicate with kafka. com.pinterest.secor.reader.LegacyKafkaMessageIterator uses
+# the old kafka consumer written scala. Its not recommended to use the legacy iterator with kafka version >= 1.0 since it
+# does not support the new broker protocols. You may face significant performance degradation on your brokers if you use it
+kafka.message.iterator.className=com.pinterest.secor.reader.SecorKafkaMessageIterator
+
+# Classname for the kafka client used by utility classes like PartitionFinalizer and Progress monitor.
+# Legacy client is kept for compatibility purposes and is deprecated
+kafka.client.className=com.pinterest.secor.common.SecorKafkaClient
+
+
+########################################
+# Optional: Kafka New Consumer Configs #
+########################################
+
+# Same as old configuration. Except accepted values are earliest and latest instead of smallest
+# and largest
+kafka.new.consumer.auto.offset.reset=earliest
+
+# Comma-separated list of topics to consume. Please note that this is not a regular expression.
+# If that's what you want, you can use "secor.kafka.topic_filter" instead.
+kafka.new.consumer.topic.list={{ get (get $.Values.secor_jobs $.Release.Name) "topic" }}
+
+kafka.new.consumer.poll.timeout.seconds=10
+kafka.new.consumer.request.timeout.ms=10000
+kafka.new.consumer.ssl.key.password=
+kafka.new.consumer.ssl.keystore.location=
+kafka.new.consumer.ssl.keystore.password=
+kafka.new.consumer.ssl.truststore.location=
+kafka.new.consumer.ssl.truststore.password=
+kafka.new.consumer.isolation.level=
+kafka.new.consumer.max.poll.interval.ms=
+kafka.new.consumer.max.poll.records=
+kafka.new.consumer.sasl.client.callback.handler.class=
+kafka.new.consumer.sasl.jaas.config=
+kafka.new.consumer.sasl.kerberos.service.name=
+kafka.new.consumer.sasl.login.callback.handler.class=
+kafka.new.consumer.sasl.login.class=
+kafka.new.consumer.sasl.mechanism=
+kafka.new.consumer.security.protocol=
+kafka.new.consumer.ssl.enabled.protocols=
+kafka.new.consumer.ssl.keystore.type=
+kafka.new.consumer.ssl.protocol=
+kafka.new.consumer.ssl.provider=
+kafka.new.consumer.ssl.truststore.type=
+kafka.new.consumer.partition.assignment.strategy.class=
+
+########################################
+# End: Kafka New Consumer Configs      #
+########################################
+
+kafka.fetch.max.bytes=
\ No newline at end of file
diff --git a/kubernetes/helm_charts/secor/config/secor.partition.properties b/kubernetes/helm_charts/secor/config/secor.partition.properties
new file mode 100644
index 0000000000..743e1bab86
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/secor.partition.properties
@@ -0,0 +1,67 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include=secor.properties
+include=secor.azure.properties
+
+# Name of the Kafka consumer group.
+secor.kafka.group={{ get (get $.Values.secor_jobs $.Release.Name) "consumer_group" }}
+
+# Parser class that extracts s3 partitions from consumed messages.
+secor.message.parser.class={{ get (get $.Values.secor_jobs $.Release.Name) "message_parser" }}
+
+# S3 path where sequence files are stored.
+secor.s3.path=
+
+# Swift path where sequence files are stored.
+secor.swift.path=secor_dev/partition
+
+# Azure path where sequence files are stored.
+secor.azure.path={{ get (get $.Values.secor_jobs $.Release.Name) "base_path" }}
+
+# Local path where sequence files are stored before they are uploaded to s3.
+secor.local.path=/mnt/data/{{ get (get $.Values.secor_jobs $.Release.Name) "service_name" }}/message_logs/partition
+
+# Port of the Ostrich server.
+#ostrich.port=
+
+# Secor custom properties
+
+# Partition Date Output format. This is used along with PatternDateMessageParser. Defaults to 'yyyy-MM-dd' *New*
+secor.partition.output_dt_format=yyyy-MM-dd
+
+secor.partition.prefix.enable={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_enabled" }}
+# Name of field that contains timestamp for JSON, MessagePack, or Thrift message parser. (1405970352123)
+secor.partition.prefix.identifier={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_key" }}
+
+secor.partition.prefix.mapping={{ get (get $.Values.secor_jobs $.Release.Name) "partition_prefix_mapping" }}
+
+secor.max.file.age.policy=oldest
+
+# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz.
+# Available placeholders are 
+# topic - The topic name the data is being fetched
+# partition - The partition name
+# generation - Generation
+# kafkaPartition - The kafka partition
+# fmOffset - First Message offset in the file.
+# randomHex - A 4 character random hex to append to the file name
+# currentTimestamp - Time of upload in epoch format
+# currentTime - Time of upload in HH-mm format
+# currentDate - Time of upload in YYYYMMDD format
+# folder - Folder to use based on message id map lookup
+secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }}
+
+secor.partition.message.channel.identifier={{ get (get $.Values.secor_jobs $.Release.Name) "message_channel_identifier" }}
diff --git a/kubernetes/helm_charts/secor/config/secor.properties b/kubernetes/helm_charts/secor/config/secor.properties
new file mode 100644
index 0000000000..6f2876d1de
--- /dev/null
+++ b/kubernetes/helm_charts/secor/config/secor.properties
@@ -0,0 +1,40 @@
+include=secor.common.properties
+
+############
+# MUST SET #
+############
+
+# Fill the section which fits your needs
+###############
+#  Using S3   #
+###############
+
+# Name of the s3 bucket where log files are stored.
+secor.s3.bucket=
+
+###############
+# Using Swift #
+###############
+
+# Boolean variable which determines if each topic will be uploaded to different container
+# The Containers for the topics will be Created automatically
+# If true, then the setting "secor.swift.container" will be ignored
+secor.swift.containers.for.each.topic=false
+
+# Name of swift container where log files are stored.
+secor.swift.container=logsContainer
+
+################
+# END MUST SET #
+################
+
+kafka.seed.broker.host={{ get (get $.Values.secor_jobs $.Release.Name) "kafka_broker_host" }}
+kafka.seed.broker.port=9092
+
+zookeeper.quorum={{ get (get $.Values.secor_jobs $.Release.Name) "zookeeper_quorum" }}
+
+# Upload policies.
+# 10K
+secor.max.file.size.bytes={{ get (get $.Values.secor_jobs $.Release.Name) "max_file_size" }}
+# 10 seconds
+secor.max.file.age.seconds={{ get (get $.Values.secor_jobs $.Release.Name) "max_file_age" }}
diff --git a/kubernetes/helm_charts/secor/templates/_helpers.tpl b/kubernetes/helm_charts/secor/templates/_helpers.tpl
new file mode 100644
index 0000000000..a63432d8bf
--- /dev/null
+++ b/kubernetes/helm_charts/secor/templates/_helpers.tpl
@@ -0,0 +1,51 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "secor.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{- define "name" -}}
+{{- default .Release.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "secor.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "secor.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+App config names
+*/}}
+{{- define "secor.appConfigName" -}}
+{{- $.Release.Name -}}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "secor.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "secor.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
diff --git a/kubernetes/helm_charts/secor/templates/secor-config.yaml b/kubernetes/helm_charts/secor/templates/secor-config.yaml
new file mode 100644
index 0000000000..f3d9cf3a55
--- /dev/null
+++ b/kubernetes/helm_charts/secor/templates/secor-config.yaml
@@ -0,0 +1,23 @@
+{{- range $job_name, $job_config :=  .Values.secor_jobs }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ $.Release.Name }}-config
+  namespace: {{ $.Values.namespace }}
+  labels:
+    app: {{ $.Release.Name }}-config
+    chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
+    release: {{ $.Release.Name }}
+    heritage: {{ $.Release.Service }}
+data:
+  secor.properties: |+
+{{- (tpl ($.Files.Get "config/secor.properties") $) | nindent 4 }}
+  secor.common.properties: |+
+{{- (tpl ($.Files.Get "config/secor.common.properties") $) | nindent 4 }}
+  secor.azure.properties: |+
+{{- (tpl ($.Files.Get "config/secor.azure.properties") $) | nindent 4 }}
+  secor.partition.properties: |+
+{{- (tpl ($.Files.Get "config/secor.partition.properties") $) | nindent 4 }}
+  log4j.properties: |+
+{{ ($.Files.Glob "config/log4j*.properties").AsConfig | nindent 4 }}
+{{- end }}
diff --git a/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml b/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml
new file mode 100644
index 0000000000..bc20b514e1
--- /dev/null
+++ b/kubernetes/helm_charts/secor/templates/secor-statefulset.yaml
@@ -0,0 +1,78 @@
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  annotations:
+    configmap.reloader.stakater.com/reload: {{ .Release.Name }}-config
+  name: {{ .Release.Name }}
+  namespace: {{ .Values.namespace }}
+  labels:
+    app: secor
+    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+spec:
+  serviceName: secor
+  updateStrategy:
+    type: RollingUpdate
+  replicas: {{ get (get $.Values.secor_jobs $.Release.Name) "replicas" }}
+  selector:
+    matchLabels:
+      app: secor
+      release: {{ .Release.Name }}
+  template:
+    metadata:
+      labels:
+        app: secor
+        release: {{ .Release.Name }}
+    spec:
+      imagePullSecrets:
+      - name: {{ .Values.imagepullsecrets }}
+      initContainers:
+      - name: change-pv-dir-ownership
+        image: alpine:3
+        command: ["/bin/sh", "-c"]
+        args: ["chown -R 9999:9999 /mnt/data"]
+        volumeMounts:
+        - name: data-path
+          mountPath: /mnt/data
+      containers:
+        - name: {{ .Release.Name }}-secor
+          image: {{ .Values.image.prefix | default "" }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          env:
+            - name: "CONFIG_FILE"
+              value: "/opt/secor/config/secor.partition.properties"
+            - name: "LOG4J_CONFIGURATION"
+              value: "/opt/secor/config/log4j.docker.properties"
+
+          volumeMounts:
+           - mountPath: /opt/secor/config
+             name: secor-config
+           - name: "data-path"
+             mountPath: /mnt/data
+          resources:
+            requests:
+              cpu: {{ get (get (get $.Values.secor_jobs $.Release.Name) "requests") "cpu" }}
+              memory: {{ get (get (get $.Values.secor_jobs $.Release.Name) "requests") "memory"}}
+          securityContext:
+            runAsUser: 9999
+
+      volumes:
+        - configMap:
+            name: {{ .Release.Name }}-config
+          name: secor-config
+
+  volumeClaimTemplates:
+    - metadata:
+        name: data-path
+        labels:
+          app: secor
+      spec:
+        storageClassName: "{{- .Values.storageClass }}"
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: {{ get (get (get $.Values.secor_jobs $.Release.Name) "storage") "size" }}
+
diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2
new file mode 100644
index 0000000000..4aa2e0ee83
--- /dev/null
+++ b/kubernetes/helm_charts/secor/values.j2
@@ -0,0 +1,63 @@
+azure_account: "{{ sunbird_private_storage_account_name }}"
+azure_secret: "{{ sunbird_private_storage_account_key }}"
+azure_container_name: "telemetry-data-store"
+
+namespace: {{ secor_namespace }}
+storageClass: {{ secor_storage_class | default('default') }}
+imagepullsecrets: {{ imagepullsecrets }}
+
+secor_jobs:
+{% for item in secor_job_list %}
+  {{ item }}:
+    replicas: "{{ secor_jobs['%s'|format(item)].replicas }}"
+    consumer_group: "{{ secor_jobs['%s'|format(item)].consumer_group }}"
+    service_name: "{{ secor_jobs['%s'|format(item)].service_name }}"
+    base_path: "{{ secor_jobs['%s'|format(item)].base_path }}"
+    timestamp_key: "{{ secor_jobs['%s'|format(item)].timestamp_key }}"
+    fallback_timestamp_key: "{{ secor_jobs['%s'|format(item)].fallback_timestamp_key }}"
+    topic: "{{ secor_jobs['%s'|format(item)].topic }}"
+    kafka_broker_host: "{{ secor_jobs['%s'|format(item)].kafka_broker_host }}"
+    zookeeper_quorum: "{{ secor_jobs['%s'|format(item)].zookeeper_quorum }}"
+    max_file_size: "{{ secor_jobs['%s'|format(item)].max_file_size }}"
+    max_file_age: "{{ secor_jobs['%s'|format(item)].max_file_age }}"
+    partition_prefix_enabled: "{{ secor_jobs['%s'|format(item)].partition_prefix_enabled }}"
+    partition_prefix_key: "{{ secor_jobs['%s'|format(item)].partition_prefix_key }}"
+    partition_prefix_mapping: '{{ secor_jobs['%s'|format(item)].partition_prefix_mapping }}'
+    message_channel_identifier: "{{ secor_jobs['%s'|format(item)].message_channel_identifier }}"
+    output_file_pattern: "{{ secor_jobs['%s'|format(item)].output_file_pattern }}"
+    message_parser: "{{ secor_jobs['%s'|format(item)].message_parser }}"
+    storage:
+      size: "{{ secor_jobs['%s'|format(item)].storage.size }}"
+    requests:
+      cpu: "{{ secor_jobs['%s'|format(item)].requests.cpu }}"
+      memory: "{{ secor_jobs['%s'|format(item)].requests.memory }}"
+    lag_threshold_warning: "{{ secor_jobs['%s'|format(item)].lag_threshold_warning }}"
+    lag_threshold_critical: "{{ secor_jobs['%s'|format(item)].lag_threshold_critical }}"
+{% endfor %}
+    
+
+image:
+  prefix: {{ dockerhub }}
+  repository: {{ image_repository }}
+  tag: {{ image_tag }}
+  pullPolicy: {{ image_pullPolicy }}
+
+exporter:
+  image:
+    repository: prom/statsd-exporter
+    tag: latest
+    pullPolicy: IfNotPresent
+
+prometheus_rule_selector_app: prometheus-operator
+prometheus_rule_selector_release: prometheus-operator
+
+# If you enable this, secor lag alert rules will be created in the flink cluster.
+# In our case the consumer group lag metrics available in core prometheus.
+# So we need to create the secor lag alert rule in core prometheus.
+# By adding this condition we are avoiding creating the secor lag alert rule in flink cluster. 
+alertrules:
+  enabled: false
+
+# This condition is whether to create the secor lag alert rule or not.
+secor_alertrule:
+  enabled: {{ secor_alertrule_enabled | lower}}
diff --git a/kubernetes/pipelines/bootstrap/Jenkinsfile b/kubernetes/pipelines/bootstrap/Jenkinsfile
new file mode 100644
index 0000000000..e70b782daf
--- /dev/null
+++ b/kubernetes/pipelines/bootstrap/Jenkinsfile
@@ -0,0 +1,46 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }  
+            checkout scm
+        }
+
+        stage('deploy') {
+            values = [:]
+            envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+            module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+            jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            ansiblePlaybook = "$currentWs/kubernetes/ansible/bootstrap_minimal.yaml"
+            ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+            values.put('currentWs', currentWs)
+            values.put('env', envDir)
+            values.put('module', module)
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            ansible_playbook_run(values)
+            currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/kubernetes/pipelines/build/Jenkinsfile b/kubernetes/pipelines/build/Jenkinsfile
index 1660222db2..52004a4d94 100644
--- a/kubernetes/pipelines/build/Jenkinsfile
+++ b/kubernetes/pipelines/build/Jenkinsfile
@@ -14,19 +14,10 @@ node('build-slave') {
                 } else
                     println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
             }
-            cleanWs()
-            if (params.github_release_tag == "") {
-                checkout scm
-                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
-                branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim()
-                build_tag = branch_name + "_" + commit_hash + "_" + env.BUILD_NUMBER
-                println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL)
-            } else {
-                def scmVars = checkout scm
-                checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
-                build_tag = params.github_release_tag + "_" + env.BUILD_NUMBER
-                println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag specified, building from tag: " + params.github_release_tag + ANSI_NORMAL)
-            }
+            cleanWs()            
+            checkout scm
+            commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+            build_tag = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
             echo "build_tag: " + build_tag
 
             stage('Build') {
diff --git a/kubernetes/pipelines/build/auto_build_deploy b/kubernetes/pipelines/build/auto_build_deploy
new file mode 100644
index 0000000000..bae59da55b
--- /dev/null
+++ b/kubernetes/pipelines/build/auto_build_deploy
@@ -0,0 +1,66 @@
+@Library('deploy-conf') _
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            withEnv(["JAVA_HOME=${JAVA11_HOME}"]) {
+                stage('Checkout') {
+                    tag_name = env.JOB_NAME.split("/")[-1]
+                    pre_checks()
+                    if (!env.hub_org) {
+                        println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
+                        error 'Please resolve the errors and rerun..'
+                    } else
+                        println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
+                }
+                cleanWs()
+                def scmVars = checkout scm
+                checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$tag_name"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
+                build_tag = tag_name + "_" + env.BUILD_NUMBER
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                artifact_version = tag_name + "_" + commit_hash
+                echo "build_tag: " + build_tag
+
+                //   stage Build
+                         env.NODE_ENV = "build"
+                         print "Environment will be : ${env.NODE_ENV}"
+                         dir('data-pipeline-flink'){
+                         sh '/opt/apache-maven-3.6.3/bin/mvn3.6 clean install -DskipTests'
+                }        
+
+                //   stage Package
+                         dir('data-pipeline-flink/sunbird-dp-distribution') {
+                         sh "/opt/apache-maven-3.6.3/bin/mvn3.6 package -Pbuild-docker-image -Drelease-version=${build_tag}"
+                }
+
+
+                //   stage Retagging
+                         sh """
+                            docker tag sunbird-datapipeline:${build_tag} ${hub_org}/sunbird-datapipeline:${build_tag}
+                            echo {\\"image_name\\" : \\"sunbird-datapipeline\\", \\"image_tag\\" : \\"${build_tag}\\", \\"node_name\\" : \\"${env.NODE_NAME}\\"} > metadata.json
+                            """
+
+               //   stage ArchiveArtifacts
+                        archiveArtifacts "metadata.json"
+                        currentBuild.description = "${build_tag}"
+
+
+            }
+            currentBuild.result = "SUCCESS"
+            slack_notify(currentBuild.result, tag_name)
+            email_notify()
+            auto_build_deploy()
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()
+        throw err
+    }
+}
diff --git a/kubernetes/pipelines/deploy/Jenkinsfile b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile
similarity index 100%
rename from kubernetes/pipelines/deploy/Jenkinsfile
rename to kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile
diff --git a/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop
new file mode 100644
index 0000000000..75bcc043f3
--- /dev/null
+++ b/kubernetes/pipelines/deploy/flink-jobs/Jenkinsfile.startstop
@@ -0,0 +1,51 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/kubernetes/ansible/flink_jobs_stop_start.yml"
+                ansibleExtraArgs = "--extra-vars \"job_names=${params.job_names}\" --tags ${params.action} --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+      //  slack_notify(currentBuild.result)
+      //  email_notify()
+    }
+}
diff --git a/kubernetes/pipelines/deploy/secor/Jenkinsfile b/kubernetes/pipelines/deploy/secor/Jenkinsfile
new file mode 100644
index 0000000000..5d10e799fd
--- /dev/null
+++ b/kubernetes/pipelines/deploy/secor/Jenkinsfile
@@ -0,0 +1,53 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }  
+            checkout scm
+        }
+
+        stage('deploy') {
+            values = docker_params()
+            jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim().toLowerCase()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            ansiblePlaybook = "$currentWs/kubernetes/ansible/secor_deploy.yaml"
+            ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/secor job_names_to_deploy=${params.job_names_to_deploy} image_tag=$values.image_tag\" --skip-tags alertrule --vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+            values.put('currentWs', currentWs)
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            ansible_playbook_run(values)
+            archiveArtifacts 'metadata.json'
+            currentBuild.description = "Image: ${values.image_tag}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        if (params.monitoring_enabled) {
+            ansiblePlaybook = "${currentWs}/kubernetes/ansible/secor_deploy.yaml"
+            ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/secor job_names_to_deploy=${params.job_names_to_deploy}\" --skip-tags secor_deploy --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            println values
+            ansible_playbook_run(values)
+            } else {
+                     echo "Alert rule creation skipped"
+          }
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/kubernetes/pipelines/monitoring/Jenkinsfile b/kubernetes/pipelines/monitoring/Jenkinsfile
index 8f95cdd3e3..1e9f2d4935 100644
--- a/kubernetes/pipelines/monitoring/Jenkinsfile
+++ b/kubernetes/pipelines/monitoring/Jenkinsfile
@@ -24,7 +24,7 @@ node() {
             jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim().toLowerCase()
             currentWs = sh(returnStdout: true, script: 'pwd').trim()
             ansiblePlaybook = "$currentWs/kubernetes/ansible/monitoring.yaml"
-            ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/$jobName kubeconfig_path=${params.kubeconfigpath} release_name=$jobName\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+            ansibleExtraArgs = "--extra-vars \"chart_path=${currentWs}/kubernetes/helm_charts/$jobName release_name=$jobName\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
             values.put('currentWs', currentWs)
             values.put('env', envDir)
             values.put('module', module)
diff --git a/kubernetes/pipelines/provision/druid/Jenkinsfile b/kubernetes/pipelines/provision/druid/Jenkinsfile
new file mode 100644
index 0000000000..c7ce8493d4
--- /dev/null
+++ b/kubernetes/pipelines/provision/druid/Jenkinsfile
@@ -0,0 +1,50 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }  
+            checkout scm
+        }
+
+        stage('deploy') {
+            values = [:]
+            envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+            module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+            jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            ansiblePlaybook = "$currentWs/kubernetes/ansible/druid_provision.yml"
+            ansibleExtraArgs = "--extra-vars \"chart_base_path=${currentWs}/kubernetes/helm_charts cluster_type=${params.druid_cluster_type}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+            values.put('currentWs', currentWs)
+            values.put('env', envDir)
+            values.put('module', module)
+            values.put('jobName', jobName)
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            println values
+            ansible_playbook_run(values)
+            currentBuild.result = "SUCCESS"
+            currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        }
+
+        summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/kubernetes/pipelines/provision/kafka/Jenkinsfile b/kubernetes/pipelines/provision/kafka/Jenkinsfile
new file mode 100644
index 0000000000..6a2f05a7b7
--- /dev/null
+++ b/kubernetes/pipelines/provision/kafka/Jenkinsfile
@@ -0,0 +1,50 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }  
+            checkout scm
+        }
+
+        stage('deploy') {
+            values = [:]
+            envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+            module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+            jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            ansiblePlaybook = "$currentWs/kubernetes/ansible/kafka_provision.yaml"
+            ansibleExtraArgs = "--extra-vars \"chart_base_path=${currentWs}/kubernetes/helm_charts\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+            values.put('currentWs', currentWs)
+            values.put('env', envDir)
+            values.put('module', module)
+            values.put('jobName', jobName)
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            println values
+            ansible_playbook_run(values)
+            currentBuild.result = "SUCCESS"
+            currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        }
+
+        summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/pipelines/backup/managed-postgres-backup/Jenkinsfile b/pipelines/backup/managed-postgres-backup/Jenkinsfile
index ec4a90d298..37216adb18 100644
--- a/pipelines/backup/managed-postgres-backup/Jenkinsfile
+++ b/pipelines/backup/managed-postgres-backup/Jenkinsfile
@@ -26,7 +26,7 @@ node() {
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
                 ansiblePlaybook = "${currentWs}/ansible/postgres-managed-service-backup.yml"
-                ansibleExtraArgs = "--tags postgres-azure-managed-service --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansibleExtraArgs = "--tags postgres-managed-service --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/backup/redis-restore/Jenkinsfile b/pipelines/backup/redis-restore/Jenkinsfile
index fd7ae88f7d..adfae91886 100644
--- a/pipelines/backup/redis-restore/Jenkinsfile
+++ b/pipelines/backup/redis-restore/Jenkinsfile
@@ -26,7 +26,7 @@ node("ops-slave") {
                 module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 ansiblePlaybook = "${currentWs}/ansible/redis-restore.yml"
-		ansibleExtraArgs = "--extra-vars \"redis_restore_file_name=${redis_restore_file_name}\" --vault-password-file  /var/lib/jenkins/secrets/vault-pass"
+		ansibleExtraArgs = "--extra-vars \"redis_restore_file_name=${redis_restore_file_name} redis_restore_process=${redis_restore_process}\" --vault-password-file  /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile b/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile
index e5433ab0df..50895c39a0 100644
--- a/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile
+++ b/pipelines/deploy/PopulatePSQL-ConsumerChannelMapping/Jenkinsfile
@@ -25,8 +25,8 @@ node() {
                 module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/postgres-provision.yml"
-                ansibleExtraArgs = "--connection=local --extra-vars \"consumer_id=${params.Consumer_id} channel=${params.Channel_id} status=${params.status} created_by=${params.Created_by}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansiblePlaybook = "${currentWs}/ansible/postgres-db-update.yml"
+                ansibleExtraArgs = "--connection=local --tags insert --extra-vars \"consumer_id=${params.Consumer_id} channel=${params.Channel_id} status=${params.status} created_by=${params.Created_by}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
@@ -35,14 +35,6 @@ node() {
                 values.put('ansibleExtraArgs', ansibleExtraArgs)
                 println values
                 ansible_playbook_run(values)
-                
-
-                ansiblePlaybook = "${currentWs}/ansible/lpa_api_deploy.yml"
-                ansibleExtraArgs = "--tags refresh-cache --vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
                 currentBuild.result = "SUCCESS"
                 currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
             }
diff --git a/pipelines/deploy/analytics-api/Jenkinsfile b/pipelines/deploy/analytics-api/Jenkinsfile
deleted file mode 100644
index e0e164d2cd..0000000000
--- a/pipelines/deploy/analytics-api/Jenkinsfile
+++ /dev/null
@@ -1,63 +0,0 @@
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-            ansiColor('xterm') {
-                values = lp_dp_params()
-                stage('get artifact') {
-                    currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                    artifact = values.artifact_name + ":" + values.artifact_version
-                    values.put('currentWs', currentWs)
-                    values.put('artifact', artifact)
-                    artifact_download(values)
-                }
-                stage('deploy artifact'){
-                    sh """
-                       unzip ${artifact}
-                       mv analytics-api-2.0-dist.zip ansible
-                    """
-                    ansiblePlaybook = "${currentWs}/ansible/lpa_api_deploy.yml"
-                    ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                    values.put('ansiblePlaybook', ansiblePlaybook)
-                    values.put('ansibleExtraArgs', ansibleExtraArgs)
-                    println values
-                    ansible_playbook_run(values)
-                    archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
-                    archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
-                    ansiblePlaybook = "${currentWs}/ansible/analyticsapi_logstash.yml"
-                    ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                    values.put('ansiblePlaybook', ansiblePlaybook)
-                    values.put('ansibleExtraArgs', ansibleExtraArgs)
-                    println values
-                    ansible_playbook_run(values)
-                    currentBuild.result = "SUCCESS"
-                    currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-                }
-            }
-             summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-}
diff --git a/pipelines/deploy/content-snapshot-indexer/Jenkinsfile b/pipelines/deploy/content-snapshot-indexer/Jenkinsfile
index 74299bc1bf..c4591aa4aa 100644
--- a/pipelines/deploy/content-snapshot-indexer/Jenkinsfile
+++ b/pipelines/deploy/content-snapshot-indexer/Jenkinsfile
@@ -30,11 +30,11 @@ node() {
                 stage('deploy artifact'){
                     sh """
                        unzip ${artifact}
-                       mv adhoc-jobs-1.0.jar ansible/roles/content-snapshot-indexer/files/
-                       mv adhoc-jobs-1.0-distribution.tar.gz ansible/roles/content-snapshot-indexer/files/
+                       mv etl-jobs-1.0.jar ansible/roles/content-snapshot-indexer/files/
+                       mv etl-jobs-1.0-distribution.tar.gz ansible/roles/content-snapshot-indexer/files/
                     """
                     ansiblePlaybook = "${currentWs}/ansible/content-snapshot-indexer.yml"
-                    ansibleExtraArgs = " --tags ${invoke_type} --extra-vars \"script_to_run=${params.script_to_run}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                    ansibleExtraArgs = " --tags ${invoke_type} --extra-vars \"script_to_run=${params.script_to_run} date=${params.date} identifier=${params.user_id} populate_anonymous_user=${params.populate_anonymous_user} refresh_data=${params.refresh_data}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                     values.put('ansiblePlaybook', ansiblePlaybook)
                     values.put('ansibleExtraArgs', ansibleExtraArgs)
                     println values
diff --git a/pipelines/deploy/data-products/Jenkinsfile b/pipelines/deploy/data-products/Jenkinsfile
index cd76b86290..a84081c5dd 100644
--- a/pipelines/deploy/data-products/Jenkinsfile
+++ b/pipelines/deploy/data-products/Jenkinsfile
@@ -43,9 +43,9 @@ node() {
                             echo "analytics-framework-2.0.jar exist"
                             mv analytics-framework-2.0.jar ansible
                        fi
-                       if [ -f scruid_2.11-2.3.2.jar ]; then
-                            echo "scruid_2.11-2.3.2.jar exist"
-                            mv scruid_2.11-2.3.2.jar ansible
+                       if [ -f scruid_2.12-2.5.0.jar ]; then
+                            echo "scruid_2.12-2.5.0.jar exist"
+                            mv scruid_2.12-2.5.0.jar ansible
                        fi
                     """
                     ansiblePlaybook = "--extra-vars \"remote=${params.remote}\" ${currentWs}/ansible/lpa_data-products_deploy.yml"
diff --git a/pipelines/deploy/datascience/Jenkinsfile b/pipelines/deploy/datascience/Jenkinsfile
index c6f9cdbce0..c567e434a1 100644
--- a/pipelines/deploy/datascience/Jenkinsfile
+++ b/pipelines/deploy/datascience/Jenkinsfile
@@ -33,7 +33,7 @@ node() {
                        mv dataproducts.tar.gz ansible
                     """
                     ansiblePlaybook = "${currentWs}/ansible/portal.yml"
-                    ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --tags common"
+                    ansibleExtraArgs = "--extra-vars \"date=${params.date}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass --tags ${params.tags}"
                     values.put('ansiblePlaybook', ansiblePlaybook)
                     values.put('ansibleExtraArgs', ansibleExtraArgs)
                     println values
diff --git a/pipelines/deploy/druid-anomaly-detection/Jenkinsfile b/pipelines/deploy/druid-anomaly-detection/Jenkinsfile
new file mode 100644
index 0000000000..81c60ea6b4
--- /dev/null
+++ b/pipelines/deploy/druid-anomaly-detection/Jenkinsfile
@@ -0,0 +1,57 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+            ansiColor('xterm') {
+                 values = lp_dp_params()
+                stage('get artifact') {
+                    currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                    artifact = values.artifact_name + ":" + values.artifact_version
+                    values.put('currentWs', currentWs)
+                    values.put('artifact', artifact)
+                    artifact_download(values)
+                }
+                stage('deploy artifact'){
+                    sh """
+                       unzip ${artifact}
+                       mv anomaly_detection.tar.gz ansible
+                    """
+                    ansiblePlaybook = "${currentWs}/ansible/druid_anomaly_detection.yml"
+                    ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                    values.put('ansiblePlaybook', ansiblePlaybook)
+                    values.put('ansibleExtraArgs', ansibleExtraArgs)
+                    println values
+                    ansible_playbook_run(values)
+                    currentBuild.result = "SUCCESS"
+                    currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+                    archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
+                    archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+                }
+            }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/pipelines/deploy/druid-run-report-job/Jenkinsfile b/pipelines/deploy/druid-run-report-job/Jenkinsfile
index 57cae469ac..e49e7a4b8d 100644
--- a/pipelines/deploy/druid-run-report-job/Jenkinsfile
+++ b/pipelines/deploy/druid-run-report-job/Jenkinsfile
@@ -26,7 +26,7 @@ node() {
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
                 ansiblePlaybook = "${currentWs}/ansible/druid-run-report.yml"
-                ansibleExtraArgs = "--extra-vars \"report_id=${params.report_id}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv"
+                ansibleExtraArgs = "--extra-vars \"report_id=${params.report_id} report_start_date=${params.report_start_date} report_end_date=${params.report_end_date}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile b/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile
new file mode 100644
index 0000000000..e4ea9da9b9
--- /dev/null
+++ b/pipelines/deploy/druid-summary-monthly-ingestion/Jenkinsfile
@@ -0,0 +1,51 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/druid-summary-monthly-ingestion.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -vv"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/pipelines/deploy/geolocation-db-setup/Jenkinsfile b/pipelines/deploy/geolocation-db-setup/Jenkinsfile
index dd5934c746..45c7c19ac8 100644
--- a/pipelines/deploy/geolocation-db-setup/Jenkinsfile
+++ b/pipelines/deploy/geolocation-db-setup/Jenkinsfile
@@ -1,47 +1,61 @@
 @Library('deploy-conf') _
 node() {
-   try {
-       String ANSI_GREEN = "\u001B[32m"
-       String ANSI_NORMAL = "\u001B[0m"
-       String ANSI_BOLD = "\u001B[1m"
-       String ANSI_RED = "\u001B[31m"
-       String ANSI_YELLOW = "\u001B[33m"
-         
-       stage('checkout public repo') {
-           cleanWs()
-           checkout scm
-       }
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
 
-       ansiColor('xterm') {
-           stage('deploy'){
-               values = [:]
-               envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-               module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-               jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-               currentWs = sh(returnStdout: true, script: 'pwd').trim()
-               ansiblePlaybook = "${currentWs}/ansible/lpa_provision_geo_location_db.yml"
-               ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-               values.put('currentWs', currentWs)
-               values.put('env', envDir)
-               values.put('module', module)
-               values.put('jobName', jobName)
-               values.put('ansiblePlaybook', ansiblePlaybook)
-               values.put('ansibleExtraArgs', ansibleExtraArgs)
-               println values
-               ansible_playbook_run(values)
-               currentBuild.result = "SUCCESS"
-               currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-           }
-       }
-   }
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+                println "Found .git folder. Clearing it.."
+                sh'git clean -fxd'
+            }
+            checkout scm
+        }
 
-   catch (err) {
-       currentBuild.result = "FAILURE"
-       throw err
-   }
-   finally {
-       slack_notify(currentBuild.result)
-       email_notify()
-   }
-}
+        ansiColor('xterm') {
+            values = [:]
+            envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+            module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+            jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+            currentWs = sh(returnStdout: true, script: 'pwd').trim()
+            artifact = params.geoip_zip_csv_file_name
+            values.put('currentWs', currentWs)
+            values.put('env', envDir)
+            values.put('module', module)
+            values.put('jobName', jobName)
+            values.put('artifact', artifact)
+            values.put('artifact', artifact)
+            values.put('ignore_job_parameter', true)
+        }
+
+        stage('get artifact') {
+            artifact_download(values)
+        }
 
+        stage('deploy'){
+            sh "mv ${artifact} ansible/roles/provision-geo-location-db/files"
+            ansiblePlaybook = "${currentWs}/ansible/lpa_provision_geo_location_db.yml"
+            ansibleExtraArgs = "--extra-vars \"geoip_zip_csv_file_name=${artifact}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+            values.put('ansiblePlaybook', ansiblePlaybook)
+            values.put('ansibleExtraArgs', ansibleExtraArgs)
+            println values
+            ansible_playbook_run(values)
+            currentBuild.result = "SUCCESS"
+            currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+        }
+    }
+
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/pipelines/deploy/replay-jobs/Jenkinsfile b/pipelines/deploy/replay-jobs/Jenkinsfile
index 245915d4a4..e8c247bc4f 100644
--- a/pipelines/deploy/replay-jobs/Jenkinsfile
+++ b/pipelines/deploy/replay-jobs/Jenkinsfile
@@ -37,7 +37,7 @@ node() {
                 ansible_playbook_run(values)
 
                 ansiblePlaybook = "${currentWs}/ansible/lpa_fetch_logs.yml"
-                ansibleExtraArgs = "--limit spark --tags ${params.job_type} --extra-vars \"job_id=${params.job_id} start_date=${params.start_date} end_date=${params.end_date} pause_min=${params.pause_min}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansibleExtraArgs = "--limit spark --tags ${params.job_type} --extra-vars \"job_id=${params.job_id} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} pause_min=${params.pause_min} keyword=${params.keyword}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('ansiblePlaybook', ansiblePlaybook)
                 values.put('ansibleExtraArgs', ansibleExtraArgs)
                 println values
diff --git a/pipelines/deploy/secor/Jenkinsfile.flink b/pipelines/deploy/secor/Jenkinsfile.flink
new file mode 100644
index 0000000000..3efeaf1a3d
--- /dev/null
+++ b/pipelines/deploy/secor/Jenkinsfile.flink
@@ -0,0 +1,65 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+            ansiColor('xterm') {
+                values = lp_dp_params()
+                stage('get artifact') {
+                    currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                    artifact = values.artifact_name + ":" + values.artifact_version
+                    values.put('currentWs', currentWs)
+                    values.put('artifact', artifact)
+                    artifact_download(values)
+                }
+                stage('deploy artifact'){
+                    sh """
+                       unzip ${artifact}
+                       mkdir -p ansible/artifacts/target
+                       mv secor-*.tar.gz ansible/artifacts/target
+                    """
+                    ansiblePlaybook = "${currentWs}/ansible/lpa_secor_deploy.yml"
+                    ansibleExtraArgs = " --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                    values.put('ansiblePlaybook', ansiblePlaybook)
+                    values.put('ansibleExtraArgs', ansibleExtraArgs)
+                    println values
+                    ansible_playbook_run(values)
+
+                    ansiblePlaybook = "${currentWs}/ansible/secor_telemetry_backup_deploy.yaml"
+                    ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                    values.put('ansiblePlaybook', ansiblePlaybook)
+                    values.put('ansibleExtraArgs', ansibleExtraArgs)
+                    println values
+                    ansible_playbook_run(values)
+                    currentBuild.result = "SUCCESS"
+                    currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+                    archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
+                    archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
+                }
+            }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
+
diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile
new file mode 100644
index 0000000000..9749d35b36
--- /dev/null
+++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile
@@ -0,0 +1,53 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml"
+                ansibleExtraArgs = "--tags ${params.job_type} --extra-vars \"job_id=${params.job_id} mode=${params.mode} partitions=${params.partitions} parallelisation=${params.parallelisation} start_date=${params.start_date} end_date=${params.end_date} batch_id=${params.batch_identifier} sparkMaster=${params.sparkMaster} pause_min=${params.pause_min} selected_partitions=${params.selected_partitions}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass -vvvv "
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
diff --git a/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel
new file mode 100644
index 0000000000..4b9891d62a
--- /dev/null
+++ b/pipelines/deploy/spark-cluster-deploy/Jenkinsfile.parallel
@@ -0,0 +1,53 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/spark-cluster-job-submit.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass --extra-vars \"jobs=${params.jobs_to_submit} batch_size=${params.batch_size}\" --tags config-update,${params.jobs_submit_type} -vvvv "
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+}
\ No newline at end of file
diff --git a/pipelines/deploy/yarn/Jenkinsfile b/pipelines/deploy/yarn/Jenkinsfile
deleted file mode 100644
index 185a2da8ac..0000000000
--- a/pipelines/deploy/yarn/Jenkinsfile
+++ /dev/null
@@ -1,63 +0,0 @@
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-        
-            ansiColor('xterm') {
-                 values = lp_dp_params()
-                stage('get artifact') {
-                    currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                    artifact = values.artifact_name + ":" + values.artifact_version
-                    values.put('currentWs', currentWs)
-                    values.put('artifact', artifact)
-                    artifact_download(values)
-                }
-                stage('deploy artifact'){
-                    sh """
-                       unzip ${artifact}
-                       mv distribution-*.tar.gz ansible
-                       rm -rf ansible/roles/samza-jobs/defaults/jobs
-                       mkdir  ansible/roles/samza-jobs/defaults/jobs ansible/roles/samza-jobs/defaults/allfiles
-                       chmod 777 ansible/roles/samza-jobs/defaults/jobs
-                       tar -xvf ansible/distribution-*.tar.gz -C ansible/roles/samza-jobs/defaults/allfiles/
-
-                    """
-                    ansiblePlaybook = "${currentWs}/ansible/samza_deploy.yml"
-                    ansibleExtraArgs = "--extra-vars \"job_names_to_kill=${params.job_names_to_deploy} job_workspace=${WORKSPACE}/ansible\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                    values.put('ansiblePlaybook', ansiblePlaybook)
-                    values.put('ansibleExtraArgs', ansibleExtraArgs)
-                    println values
-                    ansible_playbook_run(values)
-                    currentBuild.result = "SUCCESS"
-                    currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-                    archiveArtifacts artifacts: "${artifact}", fingerprint: true, onlyIfSuccessful: true
-                    archiveArtifacts artifacts: 'metadata.json', onlyIfSuccessful: true
-                }
-            }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-}
-
diff --git a/pipelines/ops/druid/Jenkinsfile.druidingestionalert b/pipelines/ops/druid/Jenkinsfile.druidingestionalert
index 3cdee9ac38..fe62c3b805 100644
--- a/pipelines/ops/druid/Jenkinsfile.druidingestionalert
+++ b/pipelines/ops/druid/Jenkinsfile.druidingestionalert
@@ -1,5 +1,5 @@
 @Library('deploy-conf') _
-node("ops-slave") {
+node() {
     try {
         String ANSI_GREEN = "\u001B[32m"
         String ANSI_NORMAL = "\u001B[0m"
diff --git a/pipelines/ops/druid/Jenkinsfile.druidsegmentalert b/pipelines/ops/druid/Jenkinsfile.druidsegmentalert
index 73a6fcb8ca..52f3c20f17 100644
--- a/pipelines/ops/druid/Jenkinsfile.druidsegmentalert
+++ b/pipelines/ops/druid/Jenkinsfile.druidsegmentalert
@@ -1,5 +1,5 @@
 @Library('deploy-conf') _
-node("ops-slave") {
+node() {
     try {
         String ANSI_GREEN = "\u001B[32m"
         String ANSI_NORMAL = "\u001B[0m"
diff --git a/pipelines/ops/druid/Jenkinsfile.ingestion b/pipelines/ops/druid/Jenkinsfile.ingestion
index 58a42ea69d..32658ebe91 100644
--- a/pipelines/ops/druid/Jenkinsfile.ingestion
+++ b/pipelines/ops/druid/Jenkinsfile.ingestion
@@ -1,5 +1,5 @@
 @Library('deploy-conf') _
-node("ops-slave") {
+node() {
     try {
         String ANSI_GREEN = "\u001B[32m"
         String ANSI_NORMAL = "\u001B[0m"
@@ -25,8 +25,20 @@ node("ops-slave") {
                 module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
+        // populating the vars for raw-overlord and rollup-overlord cluster respectively.
+                task_names="${params.ingestion_task_names}";
+                println(task_names);
+                List<String> task_names_list = task_names.split(',');
+                List<String> rollup_task_names = [], raw_task_names = [];
+                task_names_list.each {
+                    if(it.startsWith("rollup")) {
+                        rollup_task_names.add(it);
+                    }
+                    else
+                        raw_task_names.add(it);
+                }
                 ansiblePlaybook = "${currentWs}/ansible/druid-ingestion.yml"
-                ansibleExtraArgs = "--tags ${params.action} --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv"
+                ansibleExtraArgs = "--extra-vars \"rollup_ingestion_task_names=${rollup_task_names.join(",")} raw_ingestion_task_names=${raw_task_names.join(",")}\" --tags populate_var,${params.action} --vault-password-file /var/lib/jenkins/secrets/vault-pass -vv"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/provision/analytics-api/Jenkinsfile b/pipelines/provision/analytics-api/Jenkinsfile
deleted file mode 100644
index 7c77425079..0000000000
--- a/pipelines/provision/analytics-api/Jenkinsfile
+++ /dev/null
@@ -1,53 +0,0 @@
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/lpa_api_provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-
-}
\ No newline at end of file
diff --git a/pipelines/provision/cassandra/Jenkinsfile b/pipelines/provision/cassandra/Jenkinsfile
deleted file mode 100644
index 36ecc9dd9b..0000000000
--- a/pipelines/provision/cassandra/Jenkinsfile
+++ /dev/null
@@ -1,53 +0,0 @@
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/lpa_cassandra_provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-
-}
\ No newline at end of file
diff --git a/pipelines/provision/druid/Jenkinsfile b/pipelines/provision/druid/Jenkinsfile
index 4895b6536d..a234c28d3a 100644
--- a/pipelines/provision/druid/Jenkinsfile
+++ b/pipelines/provision/druid/Jenkinsfile
@@ -26,7 +26,7 @@ node() {
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
                 ansiblePlaybook = "${currentWs}/ansible/druid-provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansibleExtraArgs = "--extra-vars \"remote=${params.remote} deploy=${params.service}\" --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/provision/graphite-exporter/Jenkinsfile b/pipelines/provision/graphite-exporter/Jenkinsfile
new file mode 100644
index 0000000000..99742d268e
--- /dev/null
+++ b/pipelines/provision/graphite-exporter/Jenkinsfile
@@ -0,0 +1,51 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                ansiblePlaybook = "${currentWs}/ansible/graphite-exporter.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = 'SUCCESS'
+                currentBuild.description = "Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = 'FAILURE'
+        throw err
+    }    
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+} 
diff --git a/pipelines/provision/kibana/Jenkinsfile b/pipelines/provision/kibana/Jenkinsfile
deleted file mode 100644
index 791994178a..0000000000
--- a/pipelines/provision/kibana/Jenkinsfile
+++ /dev/null
@@ -1,54 +0,0 @@
-
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/kibana_provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-
-}
\ No newline at end of file
diff --git a/pipelines/provision/postgres-db-update/Jenkinsfile b/pipelines/provision/postgres-db-update/Jenkinsfile
new file mode 100644
index 0000000000..3fbb046008
--- /dev/null
+++ b/pipelines/provision/postgres-db-update/Jenkinsfile
@@ -0,0 +1,53 @@
+@Library('deploy-conf') _
+node() {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        stage('checkout public repo') {
+            folder = new File("$WORKSPACE/.git")
+            if (folder.exists())
+            {
+               println "Found .git folder. Clearing it.."
+               sh'git clean -fxd'
+            }
+            checkout scm
+
+        }
+
+        ansiColor('xterm') {
+            stage('deploy'){
+                values = [:]
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                ansiblePlaybook = "${currentWs}/ansible/postgres-db-update.yml"
+                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+                currentBuild.result = "SUCCESS"
+                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
+            }
+        }
+         summary()
+     }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+    finally {
+        slack_notify(currentBuild.result)
+        email_notify()
+    }
+
+} 
diff --git a/pipelines/provision/postgres-managed/Jenkinsfile b/pipelines/provision/postgres-managed/Jenkinsfile
deleted file mode 100644
index d98eee299e..0000000000
--- a/pipelines/provision/postgres-managed/Jenkinsfile
+++ /dev/null
@@ -1,53 +0,0 @@
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                ansiblePlaybook = "${currentWs}/ansible/postgres-manage.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass -v"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-
-} 
diff --git a/pipelines/provision/redis/Jenkinsfile b/pipelines/provision/redis/Jenkinsfile
index 8b4f864fae..95b2a1fc7d 100644
--- a/pipelines/provision/redis/Jenkinsfile
+++ b/pipelines/provision/redis/Jenkinsfile
@@ -26,7 +26,7 @@ node() {
                 jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
                 currentWs = sh(returnStdout: true, script: 'pwd').trim()
                 ansiblePlaybook = "${currentWs}/ansible/redis_provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                ansibleExtraArgs = "--tags ${params.redis_type} --vault-password-file /var/lib/jenkins/secrets/vault-pass"
                 values.put('currentWs', currentWs)
                 values.put('env', envDir)
                 values.put('module', module)
diff --git a/pipelines/provision/spark/Jenkinsfile.delete b/pipelines/provision/spark/Jenkinsfile.delete
new file mode 100644
index 0000000000..93aed171cb
--- /dev/null
+++ b/pipelines/provision/spark/Jenkinsfile.delete
@@ -0,0 +1,51 @@
+@Library('deploy-conf') _
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            stage('Checkout') { 
+                checkout scm            
+            }
+            
+            stage('copy cluster creation script') {
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/azure-hdinsight-spark.provision.yml"
+                ansibleExtraArgs = "--extra-vars \"azure_resource_group=${params.resource_group} subscription_id=${env.subscription_id} tenant_id=${env.tenant_id} cluster_state=${params.type}\" --tags copy-script --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+            }
+            stage('delete spark HDinsight cluster') {
+                storage_container=params.storage_container
+                withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) {
+                   sh '''
+                   cd /tmp
+                   ./delete-cluster.sh $spuser $sppass
+                   '''
+                }
+                
+               
+            }
+                
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+
+}
diff --git a/pipelines/provision/spark/Jenkinsfile.hdinsight b/pipelines/provision/spark/Jenkinsfile.hdinsight
new file mode 100644
index 0000000000..4d8f24fd39
--- /dev/null
+++ b/pipelines/provision/spark/Jenkinsfile.hdinsight
@@ -0,0 +1,56 @@
+ 
+@Library('deploy-conf') _
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            stage('Checkout') { 
+                checkout scm            
+            }
+            
+            stage('copy cluster creation script') {
+                values = [:]
+                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
+                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
+                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
+                currentWs = sh(returnStdout: true, script: 'pwd').trim()
+                ansiblePlaybook = "${currentWs}/ansible/azure-hdinsight-spark.provision.yml"
+                ansibleExtraArgs = "--extra-vars \"azure_resource_group=${params.resource_group} location=${params.location} component_version=${params.component_version} version=${params.version} headnode_size=${params.headnode_size} workernode_size=${params.workernode_size} subnet_name=${subnet_name} vnet_name=${vnet_name} workernode_count=${workernode_count} subscription_id=${env.subscription_id} tenant_id=${env.tenant_id} cluster_state=${params.type}\" --tags copy-script,copy-creation-script --vault-password-file /var/lib/jenkins/secrets/vault-pass"
+                values.put('currentWs', currentWs)
+                values.put('env', envDir)
+                values.put('module', module)
+                values.put('jobName', jobName)
+                values.put('ansiblePlaybook', ansiblePlaybook)
+                values.put('ansibleExtraArgs', ansibleExtraArgs)
+                println values
+                ansible_playbook_run(values)
+            }
+            stage('create and provision spark HDinsight cluster') {
+                storage_container=params.storage_container
+                withCredentials([usernamePassword(credentialsId: 'azure-service-principal', passwordVariable: 'sppass', usernameVariable: 'spuser')]) {
+                   sh '''
+                   currentws=$(pwd)
+                   ansibleplaybook="$currentws/ansible/azure-hdinsight-spark.provision.yml"
+                   cd /tmp
+                   ./create-cluster.sh $spuser $sppass
+                   vars="storage_container=$storage_container"
+                   export ANSIBLE_HOST_KEY_CHECKING=False
+                   ansible-playbook -i $currentws/ansible/inventory/env $ansibleplaybook --extra-vars '"'$vars'"' --tags spark-provision --vault-password-file /var/lib/jenkins/secrets/vault-pass
+                   '''
+                }   
+               
+            }
+                
+        }
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        throw err
+    }
+
+}
diff --git a/pipelines/provision/telemetry-elasticsearch/Jenkinsfile b/pipelines/provision/telemetry-elasticsearch/Jenkinsfile
deleted file mode 100644
index 43b2d89719..0000000000
--- a/pipelines/provision/telemetry-elasticsearch/Jenkinsfile
+++ /dev/null
@@ -1,60 +0,0 @@
-
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/es_telemetry_cluster_setup.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                
-                ansiblePlaybook = "${currentWs}/ansible/es_azure_plugin.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-}
diff --git a/pipelines/provision/yarn/Jenkinsfile b/pipelines/provision/yarn/Jenkinsfile
deleted file mode 100644
index 1fd44f5df9..0000000000
--- a/pipelines/provision/yarn/Jenkinsfile
+++ /dev/null
@@ -1,53 +0,0 @@
-
-@Library('deploy-conf') _
-node() {
-    try {
-        String ANSI_GREEN = "\u001B[32m"
-        String ANSI_NORMAL = "\u001B[0m"
-        String ANSI_BOLD = "\u001B[1m"
-        String ANSI_RED = "\u001B[31m"
-        String ANSI_YELLOW = "\u001B[33m"
-
-        stage('checkout public repo') {
-            folder = new File("$WORKSPACE/.git")
-            if (folder.exists())
-            {
-               println "Found .git folder. Clearing it.."
-               sh'git clean -fxd'
-            }
-            checkout scm
-
-        }
-
-        ansiColor('xterm') {
-            stage('deploy'){
-                values = [:]
-                envDir = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-3].trim()
-                module = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-2].trim()
-                jobName = sh(returnStdout: true, script: "echo $JOB_NAME").split('/')[-1].trim()
-                currentWs = sh(returnStdout: true, script: 'pwd').trim()
-                ansiblePlaybook = "${currentWs}/ansible/dp_yarn_provision.yml"
-                ansibleExtraArgs = "--vault-password-file /var/lib/jenkins/secrets/vault-pass"
-                values.put('currentWs', currentWs)
-                values.put('env', envDir)
-                values.put('module', module)
-                values.put('jobName', jobName)
-                values.put('ansiblePlaybook', ansiblePlaybook)
-                values.put('ansibleExtraArgs', ansibleExtraArgs)
-                println values
-                ansible_playbook_run(values)
-                currentBuild.result = "SUCCESS"
-                currentBuild.description = "Artifact: ${values.artifact_version}, Private: ${params.private_branch}, Public: ${params.branch_or_tag}"
-            }
-        }
-         summary()
-     }
-    catch (err) {
-        currentBuild.result = "FAILURE"
-        throw err
-    }
-    finally {
-        slack_notify(currentBuild.result)
-        email_notify()
-    }
-}
diff --git a/schemas/1.0/common.json b/schemas/1.0/common.json
deleted file mode 100644
index 57e5ed81c0..0000000000
--- a/schemas/1.0/common.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": ["1.0"]
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/1.0/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 1
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 1
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    }
-  }
-}
diff --git a/schemas/1.0/common_empty_sid_uid.json b/schemas/1.0/common_empty_sid_uid.json
deleted file mode 100644
index 91120d65ae..0000000000
--- a/schemas/1.0/common_empty_sid_uid.json
+++ /dev/null
@@ -1,39 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/1.0/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    }
-  }
-}
diff --git a/schemas/1.0/event.json b/schemas/1.0/event.json
deleted file mode 100644
index 5db20d0647..0000000000
--- a/schemas/1.0/event.json
+++ /dev/null
@@ -1,91 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_genie_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_genie_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_signup.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_session_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_session_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_launch_game.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_game_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_profile_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_view_progress.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_genie_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_game_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_genie_resume.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_misc.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_assess.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_level_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_learn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_earn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_misc.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/oe_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_create_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/ge_update_profile.json"
-    }
-  ]
-}
diff --git a/schemas/1.0/gdata.json b/schemas/1.0/gdata.json
deleted file mode 100644
index 1b603a1b7b..0000000000
--- a/schemas/1.0/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/1.0/ge_api_call.json b/schemas/1.0/ge_api_call.json
deleted file mode 100644
index 768c587382..0000000000
--- a/schemas/1.0/ge_api_call.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_api_call",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_API_CALL"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "api",
-                "msgid",
-                "isize",
-                "osize",
-                "length",
-                "err"
-              ],
-              "properties": {
-                "api": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "msgid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "isize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "osize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_create_profile.json b/schemas/1.0/ge_create_profile.json
deleted file mode 100644
index 45a90134f2..0000000000
--- a/schemas/1.0/ge_create_profile.json
+++ /dev/null
@@ -1,109 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_PROFILE"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_create_user.json b/schemas/1.0/ge_create_user.json
deleted file mode 100644
index 58ac62cef1..0000000000
--- a/schemas/1.0/ge_create_user.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_USER"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_error.json b/schemas/1.0/ge_error.json
deleted file mode 100644
index efdb81e393..0000000000
--- a/schemas/1.0/ge_error.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id",
-                "err",
-                "eventId",
-                "data"
-              ],
-              "properties": {
-                "err": {
-                  "type": "string"
-                },
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "SYSTEM","GENIE","GAME","GENIESERVICES"
-                  ]
-                },
-                "id": {
-                  "type": "string"
-                },
-               "eventId": {
-                  "type": "string"
-                },
-                "data": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_export.json b/schemas/1.0/ge_export.json
deleted file mode 100644
index e1d78df0cc..0000000000
--- a/schemas/1.0/ge_export.json
+++ /dev/null
@@ -1,91 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_EXPORT"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "telemetryEvents": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/1.0/ge_game_end.json b/schemas/1.0/ge_game_end.json
deleted file mode 100644
index be7b0a868a..0000000000
--- a/schemas/1.0/ge_game_end.json
+++ /dev/null
@@ -1,67 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "length",
-                "tmsize",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "tmsize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_game_update.json b/schemas/1.0/ge_game_update.json
deleted file mode 100644
index e7a739c1d0..0000000000
--- a/schemas/1.0/ge_game_update.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "mode",
-                "ver",
-                "size",
-                "err",
-                "gid"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "mode":{
-                   "enum": [
-                    "MDATA",
-                    "WIFI",
-                    "SDCARD",
-                    "BT",
-                    "OTHER"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_genie_end.json b/schemas/1.0/ge_genie_end.json
deleted file mode 100644
index c6fd9f367a..0000000000
--- a/schemas/1.0/ge_genie_end.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_genie_resume.json b/schemas/1.0/ge_genie_resume.json
deleted file mode 100644
index 37b6640d67..0000000000
--- a/schemas/1.0/ge_genie_resume.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_resume",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_RESUME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_genie_start.json b/schemas/1.0/ge_genie_start.json
deleted file mode 100644
index de40a918d7..0000000000
--- a/schemas/1.0/ge_genie_start.json
+++ /dev/null
@@ -1,124 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "dspec",
-                "loc"
-              ],
-              "properties": {
-                "dspec": {
-                  "id": "http://api.ekstep.org/telemetry/edata/dspec",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "os",
-                    "make",
-                    "dname",
-                    "dlocname",
-                    "mem",
-                    "id",
-                    "idisk",
-                    "edisk",
-                    "scrn",
-                    "camera",
-                    "cpu",
-                    "sims",
-                    "cap"
-                  ],
-                  "properties": {
-                    "os": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "dname": {
-                      "type": "string"
-                    },
-                    "dlocname": {
-                      "type": "string"
-                    },
-                    "make": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "id": {
-                      "type": "string"
-                    },
-                    "idisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "edisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "mem": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "sims":{
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "scrn": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "camera": {
-                      "type": "string"
-                    },
-                    "cpu": {
-                      "type": "string"
-                    },
-                    "cap": {
-                       "type": "array",
-                       "items": {
-                        "type": "string"
-                       }
-                    }
-                  }
-                },
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_genie_update.json b/schemas/1.0/ge_genie_update.json
deleted file mode 100644
index 209c139258..0000000000
--- a/schemas/1.0/ge_genie_update.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "mode",
-                "ver",
-                "size",
-                "err"
-              ],
-              "properties": {
-                "mode":{
-                   "enum": [
-                    "MDATA",
-                    "WIFI"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_import.json b/schemas/1.0/ge_import.json
deleted file mode 100644
index cbea1af453..0000000000
--- a/schemas/1.0/ge_import.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_IMPORT"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "did": {
-              "type": "string"
-            },
-            "telemetryEvents": {
-              "type": "number"
-            },
-            "userProfiles": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_interact.json b/schemas/1.0/ge_interact.json
deleted file mode 100644
index 0b0c66e399..0000000000
--- a/schemas/1.0/ge_interact.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "extype",
-                "id",
-                "uri"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "TOUCH","DRAG","DROP","PINCH","ZOOM","SHAKE","ROTATE","SPEAK","LISTEN","WRITE","DRAW","START","END","CHOOSE","ACTIVATE","OTHER"
-                  ]
-                },
-                "extype": {
-                  "type": "string"
-                },
-                "id": {
-                  "type": "string"
-                },
-                "uri": {
-                  "type": "string"
-                }
-              }
-            },
-            "ext": {
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_interrupt.json b/schemas/1.0/ge_interrupt.json
deleted file mode 100644
index 7141cacf68..0000000000
--- a/schemas/1.0/ge_interrupt.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND"
-                  ]
-                },
-               "id": {
-                  "type": "string"
-                }
-              }
-            },
-            "ext": {
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/1.0/ge_launch_game.json b/schemas/1.0/ge_launch_game.json
deleted file mode 100644
index 281dce4d12..0000000000
--- a/schemas/1.0/ge_launch_game.json
+++ /dev/null
@@ -1,65 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_launch_game",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_LAUNCH_GAME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "tmschm",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "tmschm": {
-                  "enum": [
-                    "NONE",
-                    "INTENT",
-                    "FILE"
-                  ]
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_misc.json b/schemas/1.0/ge_misc.json
deleted file mode 100644
index eaecb27168..0000000000
--- a/schemas/1.0/ge_misc.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_misc",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_MISC"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "ext"
-          ],
-          "properties": {
-            "ext": {
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_partner_data.json b/schemas/1.0/ge_partner_data.json
deleted file mode 100644
index 77faff5e5d..0000000000
--- a/schemas/1.0/ge_partner_data.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_PARTNER_DATA"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            },
-            "data": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_profile_set.json b/schemas/1.0/ge_profile_set.json
deleted file mode 100644
index 53e9be281b..0000000000
--- a/schemas/1.0/ge_profile_set.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_profile_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_PROFILE_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid",
-                "attrs"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "attrs": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_register_partner.json b/schemas/1.0/ge_register_partner.json
deleted file mode 100644
index 9d89ad8785..0000000000
--- a/schemas/1.0/ge_register_partner.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_REGISTER_PARTNER"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickey": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_service_api_call.json b/schemas/1.0/ge_service_api_call.json
deleted file mode 100644
index 76f301f308..0000000000
--- a/schemas/1.0/ge_service_api_call.json
+++ /dev/null
@@ -1,113 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_SERVICE_API_CALL"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "message": {
-              "type": "string"
-            },
-            "service": {
-              "type": "string"
-            },
-            "result": {
-              "type": "object",
-              "additionalProperties": true
-            },
-            "method": {
-              "type": "string"
-            },
-            "errorMessages": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "error": {
-              "type": "string"
-            },
-            "status": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_session_end.json b/schemas/1.0/ge_session_end.json
deleted file mode 100644
index ffce9c5942..0000000000
--- a/schemas/1.0/ge_session_end.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-               "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/1.0/ge_session_start.json b/schemas/1.0/ge_session_start.json
deleted file mode 100644
index 5cbb89d3ae..0000000000
--- a/schemas/1.0/ge_session_start.json
+++ /dev/null
@@ -1,52 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/1.0/ge_signup.json b/schemas/1.0/ge_signup.json
deleted file mode 100644
index 238ad22db9..0000000000
--- a/schemas/1.0/ge_signup.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_signup",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SIGNUP"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid",
-                "utype",
-                "loc",
-                "err"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "loc": {
-                  "type": "string"
-                },
-                "err": {
-                  "type": "string"
-                },
-                "utype": {
-                  "enum": [
-                    "CHILD",
-                    "FACILITATOR"
-                  ]
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_start_partner_session.json b/schemas/1.0/ge_start_partner_session.json
deleted file mode 100644
index 1e6b9ff3cf..0000000000
--- a/schemas/1.0/ge_start_partner_session.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_START_PARTNER_SESSION"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {}
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_stop_partner_session.json b/schemas/1.0/ge_stop_partner_session.json
deleted file mode 100644
index 6edfb65357..0000000000
--- a/schemas/1.0/ge_stop_partner_session.json
+++ /dev/null
@@ -1 +0,0 @@
-{"id":"http://api.ekstep.org/telemetry/ge_error","type":"object","required":["eid","ts","ver","gdata","sid","uid","did","edata"],"properties":{"eid":{"id":"http://api.ekstep.org/telemetry/eid","enum":["GE_STOP_PARTNER_SESSION"]},"ts":{"id":"http://api.ekstep.org/telemetry/ts","type":"string","format":"date-time"},"ver":{"id":"http://api.ekstep.org/telemetry/ver","type":"string","minLength":1},"gdata":{"type":"object","properties":{"id":{"id":"http://api.ekstep.org/telemetry/gdata/id","type":"string"},"ver":{"id":"http://api.ekstep.org/telemetry/gdata/ver","type":"string"}},"additionalProperties":false,"required":["id","ver"]},"sid":{"id":"http://api.ekstep.org/telemetry/sid","type":"string","minLength":0},"uid":{"id":"http://api.ekstep.org/telemetry/uid","type":"string","minLength":0},"did":{"id":"http://api.ekstep.org/telemetry/did","type":"string","minLength":1},"tags":{"type":"array","items":{"type":"object"}},"edata":{"id":"http://api.ekstep.org/telemetry/edata","type":"object","additionalProperties":false,"required":["eks"],"properties":{"eks":{"id":"http://api.ekstep.org/telemetry/edata/eks","type":"object","additionalProperties":true,"properties":{"length":{"type":"number"}}}}}}}
\ No newline at end of file
diff --git a/schemas/1.0/ge_update_profile.json b/schemas/1.0/ge_update_profile.json
deleted file mode 100644
index 40f90da856..0000000000
--- a/schemas/1.0/ge_update_profile.json
+++ /dev/null
@@ -1,112 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_UPDATE_PROFILE"
-      ]
-    },
-    "ts": {
-      "id": "http://api.ekstep.org/telemetry/ts",
-      "type": "string",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            },
-            "day": {
-              "type": "number"
-            },
-            "month": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/1.0/ge_view_progress.json b/schemas/1.0/ge_view_progress.json
deleted file mode 100644
index 14450c3632..0000000000
--- a/schemas/1.0/ge_view_progress.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_view_progress",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_VIEW_PROGRESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_assess.json b/schemas/1.0/oe_assess.json
deleted file mode 100644
index 0d5445173f..0000000000
--- a/schemas/1.0/oe_assess.json
+++ /dev/null
@@ -1,151 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/oe_assess",
-  "type": "object",
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {  
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ASSESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "subj",
-                "mc",
-                "qid",
-                "qtype",
-                "qlevel",
-                "pass",
-                "score",
-                "maxscore"
-              ],
-              "properties": {
-                "subj": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/subj",
-                  "enum": [
-                    "NUM",
-                    "LIT",
-                    "COG"
-                  ]
-                },
-                "mc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "qid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qid",
-                  "type": "string"
-                },
-                "qtype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qtype",
-                  "type": "string"
-                },
-                "qlevel": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qlevel",
-                  "type": "string"
-                },
-                "pass": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/pass",
-                  "enum": [
-                    "Yes",
-                    "No"
-                  ]
-                },
-                "mmc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mmc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "score": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/score",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "maxscore": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "res": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/res",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "exres": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/exres",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "length": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/length",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "exlength": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/exlength",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "atmpts": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/atmpts",
-                  "type": "number",
-                  "multipleOf": 1,
-                  "minimum": 0
-                },
-                "failedatmpts": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/failedatmpts",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/uri",
-                  "type": "string"
-                }
-              }
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ],
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ]
-}
diff --git a/schemas/1.0/oe_earn.json b/schemas/1.0/oe_earn.json
deleted file mode 100644
index 6d8c2a04e5..0000000000
--- a/schemas/1.0/oe_earn.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_earn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_EARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object"
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_end.json b/schemas/1.0/oe_end.json
deleted file mode 100644
index 2902a70fab..0000000000
--- a/schemas/1.0/oe_end.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks",
-            "ext"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length":{
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_interact.json b/schemas/1.0/oe_interact.json
deleted file mode 100644
index 210a4d6b96..0000000000
--- a/schemas/1.0/oe_interact.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "extype",
-                "id",
-                "uri"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "TOUCH",
-                    "DRAG",
-                    "DROP",
-                    "PINCH",
-                    "ZOOM",
-                    "SHAKE",
-                    "ROTATE",
-                    "SPEAK",
-                    "LISTEN",
-                    "WRITE",
-                    "DRAW",
-                    "START",
-                    "END",
-                    "CHOOSE",
-                    "ACTIVATE",
-                    "OTHER"
-                  ]
-                },
-                "extype": {
-                  "type": "string"
-                },
-                "id": {
-                  "type": "string"
-                },
-                "uri": {
-                  "type": "string"
-                }
-              }
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_interrupt.json b/schemas/1.0/oe_interrupt.json
deleted file mode 100644
index c909070c8c..0000000000
--- a/schemas/1.0/oe_interrupt.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER"
-                  ]
-                },
-               "id": {
-                  "type": "string"
-                }
-              }
-            },
-            "ext": {
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_learn.json b/schemas/1.0/oe_learn.json
deleted file mode 100644
index 348d0d2e73..0000000000
--- a/schemas/1.0/oe_learn.json
+++ /dev/null
@@ -1,89 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_learn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "topics"
-              ],
-              "properties": {
-                "topics": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/topics",
-                  "type": "array",
-                  "minItems": 1,
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                      "mc",
-                      "skill",
-                      "methods"
-                    ],
-                    "properties": {
-                      "mc": {
-                        "type": "string"
-                      },
-                      "skill": {
-                        "type": "string"
-                      },
-                      "methods": {
-                        "type": "array",
-                        "minItems": 1,
-                        "items": {
-                          "type": "string",
-                          "enum": [
-                            "PLAY",
-                            "ANSWER",
-                            "WRITE",
-                            "SPEAK",
-                            "OTHER"
-                          ]
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_level_set.json b/schemas/1.0/oe_level_set.json
deleted file mode 100644
index e575d00d24..0000000000
--- a/schemas/1.0/oe_level_set.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_level_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEVEL_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "category",
-                "current",
-                "max"
-              ],
-              "properties": {
-                "category": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/category",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "current": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/current",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "max": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/max",
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_misc.json b/schemas/1.0/oe_misc.json
deleted file mode 100644
index 46d6a95f0a..0000000000
--- a/schemas/1.0/oe_misc.json
+++ /dev/null
@@ -1,42 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_misc",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_MISC"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "ext"
-          ],
-          "properties": {
-            "ext": {
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/1.0/oe_start.json b/schemas/1.0/oe_start.json
deleted file mode 100644
index ec26c93c57..0000000000
--- a/schemas/1.0/oe_start.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ts",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/1.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks",
-            "ext"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false
-            },
-            "ext": {
-              "id": "http://api.ekstep.org/telemetry/edata/ext",
-              "type": "object"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/2.0/cdata.json b/schemas/2.0/cdata.json
deleted file mode 100644
index 4c7261b025..0000000000
--- a/schemas/2.0/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
diff --git a/schemas/2.0/common.json b/schemas/2.0/common.json
deleted file mode 100644
index 9abb81be4f..0000000000
--- a/schemas/2.0/common.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": ["2.0"]
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.0/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 1
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 1
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.0/cdata.json"
-    }
-  }
-}
diff --git a/schemas/2.0/common_empty_sid_uid.json b/schemas/2.0/common_empty_sid_uid.json
deleted file mode 100644
index 7b3c65966a..0000000000
--- a/schemas/2.0/common_empty_sid_uid.json
+++ /dev/null
@@ -1,47 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.0/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.0/cdata.json"
-    }
-  }
-}
diff --git a/schemas/2.0/event.json b/schemas/2.0/event.json
deleted file mode 100644
index 2bdbae4479..0000000000
--- a/schemas/2.0/event.json
+++ /dev/null
@@ -1,115 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_genie_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_genie_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_session_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_session_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_launch_game.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_game_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_profile_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_partner_data.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_register_partner.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_view_progress.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_genie_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_game_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_service_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_genie_resume.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_assess.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_level_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_learn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_earn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_item_response.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_navigate.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_create_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_update_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_transfer.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/ge_delete_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/oe_xapi.json"
-    }
-  ]
-}
diff --git a/schemas/2.0/gdata.json b/schemas/2.0/gdata.json
deleted file mode 100644
index 1b603a1b7b..0000000000
--- a/schemas/2.0/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/2.0/ge_api_call.json b/schemas/2.0/ge_api_call.json
deleted file mode 100644
index a16544fa84..0000000000
--- a/schemas/2.0/ge_api_call.json
+++ /dev/null
@@ -1,78 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_api_call",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_API_CALL"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "api",
-                "msgid",
-                "isize",
-                "osize",
-                "length",
-                "err"
-              ],
-              "properties": {
-                "api": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "msgid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "isize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "osize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_create_profile.json b/schemas/2.0/ge_create_profile.json
deleted file mode 100644
index 89272cf4ab..0000000000
--- a/schemas/2.0/ge_create_profile.json
+++ /dev/null
@@ -1,122 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            },
-            "day": {
-              "type": "number"
-            },
-            "month": {
-              "type": "number"
-            },
-            "board": {
-              "type": "string"
-            },
-            "medium": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_create_user.json b/schemas/2.0/ge_create_user.json
deleted file mode 100644
index 2f0391439c..0000000000
--- a/schemas/2.0/ge_create_user.json
+++ /dev/null
@@ -1,95 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_USER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_delete_profile.json b/schemas/2.0/ge_delete_profile.json
deleted file mode 100644
index 2969259e48..0000000000
--- a/schemas/2.0/ge_delete_profile.json
+++ /dev/null
@@ -1,95 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_delete_profile",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_DELETE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "uid": {
-              "type": "string"
-            },
-            "duration": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_error.json b/schemas/2.0/ge_error.json
deleted file mode 100644
index d97651605c..0000000000
--- a/schemas/2.0/ge_error.json
+++ /dev/null
@@ -1,72 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id",
-                "err",
-                "eventId",
-                "data"
-              ],
-              "properties": {
-                "err": {
-                  "type": "string"
-                },
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "SYSTEM","GENIE","GAME","GENIESERVICES"
-                  ]
-                },
-                "id": {
-                  "type": "string"
-                },
-               "eventId": {
-                  "type": "string"
-                },
-                "data": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_export.json b/schemas/2.0/ge_export.json
deleted file mode 100644
index 79ad905c30..0000000000
--- a/schemas/2.0/ge_export.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_EXPORT"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "telemetryEvents": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_feedback.json b/schemas/2.0/ge_feedback.json
deleted file mode 100644
index 05b438fb1b..0000000000
--- a/schemas/2.0/ge_feedback.json
+++ /dev/null
@@ -1,101 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "rating",
-                "context",
-                "comments"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "type": "string",
-                  "enum": [
-                    "FLAG",
-                    "SURVEY",
-                    "RATING"
-                  ]
-                },
-                "rating": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                  "type": "number"
-                },
-                "context": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "type",
-                    "id",
-                    "stageid"
-                  ],
-                  "properties": {
-                    "type": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/type",
-                      "type": "string"
-                    },
-                    "id": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/id",
-                      "type": "string"
-                    },
-                    "stageid": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid",
-                      "type": "string"
-                    }
-                  }
-                },
-                "comments": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                  "type": "string"
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_game_end.json b/schemas/2.0/ge_game_end.json
deleted file mode 100644
index 8b132c99af..0000000000
--- a/schemas/2.0/ge_game_end.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "length",
-                "tmsize",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "tmsize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_game_update.json b/schemas/2.0/ge_game_update.json
deleted file mode 100644
index deabbb1215..0000000000
--- a/schemas/2.0/ge_game_update.json
+++ /dev/null
@@ -1,78 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "mode",
-                "ver",
-                "size",
-                "err",
-                "gid"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "mode":{
-                   "enum": [
-                    "MDATA",
-                    "WIFI",
-                    "SDCARD",
-                    "BT",
-                    "OTHER"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_genie_end.json b/schemas/2.0/ge_genie_end.json
deleted file mode 100644
index 7b25a9f634..0000000000
--- a/schemas/2.0/ge_genie_end.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_genie_resume.json b/schemas/2.0/ge_genie_resume.json
deleted file mode 100644
index 638e7fd260..0000000000
--- a/schemas/2.0/ge_genie_resume.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_resume",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_RESUME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_genie_start.json b/schemas/2.0/ge_genie_start.json
deleted file mode 100644
index aa89ad4a6f..0000000000
--- a/schemas/2.0/ge_genie_start.json
+++ /dev/null
@@ -1,144 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "dspec",
-                "loc"
-              ],
-              "properties": {
-                "dspec": {
-                  "id": "http://api.ekstep.org/telemetry/edata/dspec",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "os",
-                    "make",
-                    "dname",
-                    "dlocname",
-                    "mem",
-                    "id",
-                    "idisk",
-                    "edisk",
-                    "scrn",
-                    "camera",
-                    "cpu",
-                    "sims",
-                    "cap"
-                  ],
-                  "properties": {
-                    "os": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "dname": {
-                      "type": "string"
-                    },
-                    "dlocname": {
-                      "type": "string"
-                    },
-                    "make": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "id": {
-                      "type": "string"
-                    },
-                    "idisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "edisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "mem": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "sims": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "scrn": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "camera": {
-                      "type": "string"
-                    },
-                    "cpu": {
-                      "type": "string"
-                    },
-                    "cap": {
-                      "type": "array",
-                      "items": {
-                        "type": "string"
-                      }
-                    },
-                    "mdata": {
-                      "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata",
-                      "type": "object",
-                      "additionalProperties": false,
-                      "required": [
-                        "type",
-                        "id"
-                      ],
-                      "properties": {
-                        "type": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type",
-                          "type": "string"
-                        },
-                        "id": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id",
-                          "type": "string"
-                        }
-                      }
-                    }
-                  }
-                },
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_genie_update.json b/schemas/2.0/ge_genie_update.json
deleted file mode 100644
index fb8c249ffd..0000000000
--- a/schemas/2.0/ge_genie_update.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "mode": {
-                  "enum": [
-                    "MDATA",
-                    "WIFI"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                },
-                "referrer": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/referrer",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_import.json b/schemas/2.0/ge_import.json
deleted file mode 100644
index 3bc7ea0e5e..0000000000
--- a/schemas/2.0/ge_import.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_IMPORT"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "did": {
-              "type": "string"
-            },
-            "telemetryEvents": {
-              "type": "number"
-            },
-            "userProfiles": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_interact.json b/schemas/2.0/ge_interact.json
deleted file mode 100644
index 3cfd851489..0000000000
--- a/schemas/2.0/ge_interact.json
+++ /dev/null
@@ -1,115 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "stageid",
-                "type",
-                "subtype",
-                "extype",
-                "pos",
-                "values",
-                "id",
-                "tid",
-                "uri"
-              ],
-              "properties": {
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                  },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                    "TOUCH",
-                    "SHOW",
-                    "HIDE",
-                    "DRAG",
-                    "DROP",
-                    "PINCH",
-                    "ZOOM",
-                    "SHAKE",
-                    "ROTATE",
-                    "SPEAK",
-                    "LISTEN",
-                    "WRITE",
-                    "DRAW",
-                    "START",
-                    "END",
-                    "CHOOSE",
-                    "ACTIVATE",
-                    "OTHER"
-                  ]
-                },
-                "subtype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                  "type": "string"
-                },
-                "extype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/extype",
-                  "type": "string"
-                },
-                "pos": {
-                  "id": "http://api.ekstep.org/telemetry/edata/pos",
-                  "type": "array"
-                },
-                "values": {
-                  "id": "http://api.ekstep.org/telemetry/edata/values",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "id": {
-                  "id": "http://api.ekstep.org/telemetry/edata/id",
-                  "type": "string"
-                },
-                "tid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/tid",
-                  "type": "string"
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/uri",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_interrupt.json b/schemas/2.0/ge_interrupt.json
deleted file mode 100644
index 4d2943cadc..0000000000
--- a/schemas/2.0/ge_interrupt.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND"
-                  ]
-                },
-               "id": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_launch_game.json b/schemas/2.0/ge_launch_game.json
deleted file mode 100644
index 338c5844e5..0000000000
--- a/schemas/2.0/ge_launch_game.json
+++ /dev/null
@@ -1,66 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_launch_game",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_LAUNCH_GAME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "tmschm",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "tmschm": {
-                  "enum": [
-                    "NONE",
-                    "INTENT",
-                    "FILE"
-                  ]
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_misc.json b/schemas/2.0/ge_misc.json
deleted file mode 100644
index ef37342a20..0000000000
--- a/schemas/2.0/ge_misc.json
+++ /dev/null
@@ -1,35 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_misc",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_MISC"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_partner_data.json b/schemas/2.0/ge_partner_data.json
deleted file mode 100644
index ea0a5541c9..0000000000
--- a/schemas/2.0/ge_partner_data.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_PARTNER_DATA"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            },
-            "data": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_profile_set.json b/schemas/2.0/ge_profile_set.json
deleted file mode 100644
index 552270f10e..0000000000
--- a/schemas/2.0/ge_profile_set.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_profile_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_PROFILE_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid",
-                "attrs"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "attrs": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_register_partner.json b/schemas/2.0/ge_register_partner.json
deleted file mode 100644
index d6a74b92b9..0000000000
--- a/schemas/2.0/ge_register_partner.json
+++ /dev/null
@@ -1,98 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_REGISTER_PARTNER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickey": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_service_api_call.json b/schemas/2.0/ge_service_api_call.json
deleted file mode 100644
index a316c5cd52..0000000000
--- a/schemas/2.0/ge_service_api_call.json
+++ /dev/null
@@ -1,125 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_SERVICE_API_CALL"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.0"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "message": {
-              "type": "string"
-            },
-            "service": {
-              "type": "string"
-            },
-            "result": {
-              "type": "object",
-              "additionalProperties": true
-            },
-            "method": {
-              "type": "string"
-            },
-            "errorMessages": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "error": {
-              "type": "string"
-            },
-            "status": {
-              "type": "string"
-            },
-            "mode": {
-              "type": "string",
-              "enum": [
-                "WIFI",
-                "MDATA",
-                "LOCAL",
-                ""
-              ]
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_session_end.json b/schemas/2.0/ge_session_end.json
deleted file mode 100644
index b7ddec5fa4..0000000000
--- a/schemas/2.0/ge_session_end.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-               "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_session_start.json b/schemas/2.0/ge_session_start.json
deleted file mode 100644
index 15a92d1bd3..0000000000
--- a/schemas/2.0/ge_session_start.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/ge_start_partner_session.json b/schemas/2.0/ge_start_partner_session.json
deleted file mode 100644
index 4a43f0d066..0000000000
--- a/schemas/2.0/ge_start_partner_session.json
+++ /dev/null
@@ -1,88 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_START_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {}
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_stop_partner_session.json b/schemas/2.0/ge_stop_partner_session.json
deleted file mode 100644
index c35b669642..0000000000
--- a/schemas/2.0/ge_stop_partner_session.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_STOP_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "length": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_transfer.json b/schemas/2.0/ge_transfer.json
deleted file mode 100644
index 1aafc0a6d7..0000000000
--- a/schemas/2.0/ge_transfer.json
+++ /dev/null
@@ -1,140 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_transfer",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_TRANSFER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "direction": {
-              "enum": [
-                "EXPORT",
-                "IMPORT"
-              ]
-            },
-            "datatype": {
-              "enum": [
-                "TELEMETRY",
-                "CONTENT",
-                "PROFILE",
-                "EXPLODEDCONTENT"
-              ]
-            },
-            "count": {
-              "type": "number"
-            },
-            "size": {
-              "type": "number"
-            },
-            "contents": {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "additionalProperties": true,
-                "properties": {
-                  "identifier": {
-                    "type": "string"
-                  },
-                  "origin": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "transferId": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "transferCount": {
-                    "type": "number"
-                  },
-                  "pkgVersion": {
-                    "type": "number"
-                  }
-                }
-              }
-            }
-          },
-          "required": [
-            "direction",
-            "datatype",
-            "count"
-          ]
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_update_profile.json b/schemas/2.0/ge_update_profile.json
deleted file mode 100644
index 6fa1c81aef..0000000000
--- a/schemas/2.0/ge_update_profile.json
+++ /dev/null
@@ -1,113 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_UPDATE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            },
-            "day": {
-              "type": "number"
-            },
-            "month": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.0/ge_view_progress.json b/schemas/2.0/ge_view_progress.json
deleted file mode 100644
index f1edff46ad..0000000000
--- a/schemas/2.0/ge_view_progress.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_view_progress",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_VIEW_PROGRESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_assess.json b/schemas/2.0/oe_assess.json
deleted file mode 100644
index cb682a3f94..0000000000
--- a/schemas/2.0/oe_assess.json
+++ /dev/null
@@ -1,134 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/oe_assess",
-  "type": "object",
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ASSESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "qid",
-                "pass",
-                "score",
-                "resvalues",
-                "length",
-                "exlength",
-                "params",
-                "uri",
-                "qindex"
-              ],
-              "properties": {
-                "qid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qid",
-                  "type": "string"
-                },
-                "pass": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/pass",
-                  "enum": [
-                    "Yes",
-                    "No"
-                  ]
-                },
-                "score": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/score",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                },
-                "length": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/length",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "exlength": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/exlength",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "params": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/params",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/uri",
-                  "type": "string"
-                },
-                "qindex": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qindex",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "qtitle": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                  "type": "string"
-                },
-                "qdesc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                  "type": "string"
-                },
-                "maxscore": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "mc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "mmc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mmc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ],
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ]
-}
diff --git a/schemas/2.0/oe_earn.json b/schemas/2.0/oe_earn.json
deleted file mode 100644
index 06a78b5e2c..0000000000
--- a/schemas/2.0/oe_earn.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_earn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_EARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "required": [
-                "type",
-                "points"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                     "MONEY","GEMS","POINTS"
-                  ]
-                },
-                "points": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_end.json b/schemas/2.0/oe_end.json
deleted file mode 100644
index 6a087bec32..0000000000
--- a/schemas/2.0/oe_end.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length":{
-                  "type": "number",
-                  "minimum": 0
-                },
-                "stageid":{
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_error.json b/schemas/2.0/oe_error.json
deleted file mode 100644
index e96ec14568..0000000000
--- a/schemas/2.0/oe_error.json
+++ /dev/null
@@ -1,100 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_error",
-  "type": "object",
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "env",
-                "type",
-                "stageid",
-                "objecttype",
-                "objectid",
-                "err",
-                "action",
-                "data",
-                "severity"
-              ],
-              "properties": {
-                "env": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/env",
-                  "type": "string"
-                },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "enum": [
-                    "SYSTEM",
-                    "API",
-                    "PLUGIN",
-                    "CONTENT",
-                    "ASSET",
-                    "OTHER"
-                  ]
-                },
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/stageid",
-                  "type": "string"
-                },
-                "objecttype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/objecttype",
-                  "type": "string"
-                },
-                "objectid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/objectid",
-                  "type": "string"
-                },
-                "err": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/err",
-                  "type": "string"
-                },
-                "action": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/action",
-                  "type": "string"
-                },
-                "data": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/data",
-                  "type": "string"
-                },
-                "severity": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/severity",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ],
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata"
-  ]
-}
diff --git a/schemas/2.0/oe_feedback.json b/schemas/2.0/oe_feedback.json
deleted file mode 100644
index 26e42c1386..0000000000
--- a/schemas/2.0/oe_feedback.json
+++ /dev/null
@@ -1,80 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "rating",
-                "context",
-                "comments",
-                "resvalues"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "type": "string",
-                  "enum": [
-                    "FLAG", "SURVEY", "RATING"
-                  ]
-                },
-                "rating": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                  "type": "number"
-                },
-                "context": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                  "type": "string"
-                },
-                "comments": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                  "type": "string"
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_interact.json b/schemas/2.0/oe_interact.json
deleted file mode 100644
index d81f749957..0000000000
--- a/schemas/2.0/oe_interact.json
+++ /dev/null
@@ -1,119 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "stageid",
-                "type",
-                "subtype",
-                "extype",
-                "pos",
-                "values",
-                "id",
-                "tid",
-                "uri"
-              ],
-              "properties": {
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                  },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                    "TOUCH",
-                    "SHOW",
-                    "HIDE",
-                    "DRAG",
-                    "DROP",
-                    "PINCH",
-                    "ZOOM",
-                    "SHAKE",
-                    "ROTATE",
-                    "SPEAK",
-                    "LISTEN",
-                    "WRITE",
-                    "DRAW",
-                    "START",
-                    "END",
-                    "CHOOSE",
-                    "ACTIVATE",
-                    "OTHER",
-                    "SCROLL",
-                    "HEARTBEAT"
-                  ]
-                },
-                "subtype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                  "type": "string"
-                },
-                "extype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/extype",
-                  "type": "string"
-                },
-                "pos": {
-                  "id": "http://api.ekstep.org/telemetry/edata/pos",
-                  "type": "array"
-                },
-                "values": {
-                  "id": "http://api.ekstep.org/telemetry/edata/values",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "id": {
-                  "id": "http://api.ekstep.org/telemetry/edata/id",
-                  "type": "string"
-                },
-                "tid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/tid",
-                  "type": "string"
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/uri",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_interrupt.json b/schemas/2.0/oe_interrupt.json
deleted file mode 100644
index 08fc8306a6..0000000000
--- a/schemas/2.0/oe_interrupt.json
+++ /dev/null
@@ -1,60 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "stageid"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND","RESUME"
-                  ]
-                },
-               "stageid": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_item_response.json b/schemas/2.0/oe_item_response.json
deleted file mode 100644
index 8b472bc8fa..0000000000
--- a/schemas/2.0/oe_item_response.json
+++ /dev/null
@@ -1,79 +0,0 @@
-
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ITEM_RESPONSE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "qid",
-                "type",
-                "state",
-                "resvalues"
-              ],
-              "properties": {
-                "qid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/qid",
-                  "type": "string"
-                },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                     "CHOOSE","DRAG","SELECT","MATCH","INPUT","SPEAK","WRITE"
-                  ]
-                },
-                "state": {
-                  "id": "http://api.ekstep.org/telemetry/edata/state",
-                  "type": "string",
-                  "enum": [
-                    "SELECTED", "UNSELECTED"
-                  ]
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_learn.json b/schemas/2.0/oe_learn.json
deleted file mode 100644
index 31f8fc7ce7..0000000000
--- a/schemas/2.0/oe_learn.json
+++ /dev/null
@@ -1,84 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_learn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "topics"
-              ],
-              "properties": {
-                "topics": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/topics",
-                  "type": "array",
-                  "minItems": 1,
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                      "mc",
-                      "methods"
-                    ],
-                    "properties": {
-                      "mc": {
-                        "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                        "type": "string"
-                      },
-                      "methods": {
-                        "id": "http://api.ekstep.org/telemetry/edata/eks/methods",
-                        "type": "array",
-                        "minItems": 1,
-                        "items": {
-                          "type": "string",
-                          "enum": [
-                            "PLAY",
-                            "ANSWER",
-                            "WRITE",
-                            "SPEAK",
-                            "OTHER"
-                          ]
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_level_set.json b/schemas/2.0/oe_level_set.json
deleted file mode 100644
index f30bf741d2..0000000000
--- a/schemas/2.0/oe_level_set.json
+++ /dev/null
@@ -1,67 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_level_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEVEL_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "category",
-                "current",
-                "max"
-              ],
-              "properties": {
-                "category": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/category",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "current": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/current",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "max": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/max",
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_navigate.json b/schemas/2.0/oe_navigate.json
deleted file mode 100644
index 74cb9a1b36..0000000000
--- a/schemas/2.0/oe_navigate.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_NAVIGATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "itype",
-                "stageid",
-                "stageto"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string"
-                },
-                "itype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/itype",
-                  "type": "string"
-                },
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                },
-                "stageto": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_start.json b/schemas/2.0/oe_start.json
deleted file mode 100644
index c2ac2b24e6..0000000000
--- a/schemas/2.0/oe_start.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "mode": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mode",  
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.0/oe_xapi.json b/schemas/2.0/oe_xapi.json
deleted file mode 100644
index 8b27a4ea2a..0000000000
--- a/schemas/2.0/oe_xapi.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_xapi",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_XAPI"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "xapi"
-              ],
-              "properties": {
-                "xapi": {
-                  "id": "http://api.ekstep.org/telemetry/edata/xapi",
-                  "type": "object"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/cdata.json b/schemas/2.1/cdata.json
deleted file mode 100644
index 4c7261b025..0000000000
--- a/schemas/2.1/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
diff --git a/schemas/2.1/common.json b/schemas/2.1/common.json
deleted file mode 100644
index 9a54ea88cc..0000000000
--- a/schemas/2.1/common.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.1"
-      ]
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/gdata.json"
-    },
-    "pdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/pdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 1
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 1
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "http://localhost:7070/schemas/2.1/etags.json"
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/2.1/common_empty_sid_uid.json b/schemas/2.1/common_empty_sid_uid.json
deleted file mode 100644
index 7f5fea1735..0000000000
--- a/schemas/2.1/common_empty_sid_uid.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.1"
-      ]
-    },
-    "pdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/pdata.json"
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "http://localhost:7070/schemas/2.1/etags.json"
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.1/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/2.1/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
diff --git a/schemas/2.1/etags.json b/schemas/2.1/etags.json
deleted file mode 100644
index e00c02e041..0000000000
--- a/schemas/2.1/etags.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "type": "object",
-    "properties": {
-      "app": {
-        "id": "http://api.ekstep.org/telemetry/etags/app",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/app/items",
-            "type": "string"
-        }
-      },
-      "partner": {
-        "id": "http://api.ekstep.org/telemetry/etags/partner",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/partner/items",
-            "type": "string"
-        }
-      },
-      "dims": {
-        "id": "http://api.ekstep.org/telemetry/etags/dims",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/dims/items",
-            "type": "string"
-        }
-      }
-    },
-    "additionalProperties": false
-}
diff --git a/schemas/2.1/event.json b/schemas/2.1/event.json
deleted file mode 100644
index 0cf8cc2f3c..0000000000
--- a/schemas/2.1/event.json
+++ /dev/null
@@ -1,127 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_genie_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_genie_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_session_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_session_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_launch_game.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_game_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_profile_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_partner_data.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_register_partner.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_start_partner_session.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_stop_partner_session.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_view_progress.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_genie_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_game_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_service_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_genie_resume.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_assess.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_level_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_learn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_earn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_item_response.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_navigate.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_create_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_update_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_transfer.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_delete_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_summary.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/oe_xapi.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/ge_create_user.json"
-    }
-  ]
-}
diff --git a/schemas/2.1/gdata.json b/schemas/2.1/gdata.json
deleted file mode 100644
index 1b603a1b7b..0000000000
--- a/schemas/2.1/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/2.1/ge_api_call.json b/schemas/2.1/ge_api_call.json
deleted file mode 100644
index b3b7c75d6f..0000000000
--- a/schemas/2.1/ge_api_call.json
+++ /dev/null
@@ -1,80 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_api_call",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_API_CALL"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "api",
-                "msgid",
-                "isize",
-                "osize",
-                "length",
-                "err"
-              ],
-              "properties": {
-                "api": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "msgid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "isize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "osize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_create_profile.json b/schemas/2.1/ge_create_profile.json
deleted file mode 100644
index 809f90122f..0000000000
--- a/schemas/2.1/ge_create_profile.json
+++ /dev/null
@@ -1,124 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            },
-            "day": {
-              "type": "number"
-            },
-            "month": {
-              "type": "number"
-            },
-            "board": {
-              "type": "string"
-            },
-            "medium": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_create_user.json b/schemas/2.1/ge_create_user.json
deleted file mode 100644
index e9ae8936b0..0000000000
--- a/schemas/2.1/ge_create_user.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_CREATE_USER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "loc": {
-              "type": "string"
-            },
-            "uid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_delete_profile.json b/schemas/2.1/ge_delete_profile.json
deleted file mode 100644
index 398d932319..0000000000
--- a/schemas/2.1/ge_delete_profile.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_delete_profile",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_DELETE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "uid": {
-              "type": "string"
-            },
-            "duration": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_error.json b/schemas/2.1/ge_error.json
deleted file mode 100644
index fbb84e8621..0000000000
--- a/schemas/2.1/ge_error.json
+++ /dev/null
@@ -1,74 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id",
-                "err",
-                "eventId",
-                "data"
-              ],
-              "properties": {
-                "err": {
-                  "type": "string"
-                },
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "SYSTEM","GENIE","GAME","GENIESERVICES"
-                  ]
-                },
-                "id": {
-                  "type": "string"
-                },
-               "eventId": {
-                  "type": "string"
-                },
-                "data": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_export.json b/schemas/2.1/ge_export.json
deleted file mode 100644
index 29cd66a90b..0000000000
--- a/schemas/2.1/ge_export.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_EXPORT"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "telemetryEvents": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_feedback.json b/schemas/2.1/ge_feedback.json
deleted file mode 100644
index 3517193ec3..0000000000
--- a/schemas/2.1/ge_feedback.json
+++ /dev/null
@@ -1,103 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "rating",
-                "context",
-                "comments"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "type": "string",
-                  "enum": [
-                    "FLAG",
-                    "SURVEY",
-                    "RATING"
-                  ]
-                },
-                "rating": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                  "type": "number"
-                },
-                "context": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "type",
-                    "id",
-                    "stageid"
-                  ],
-                  "properties": {
-                    "type": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/type",
-                      "type": "string"
-                    },
-                    "id": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/id",
-                      "type": "string"
-                    },
-                    "stageid": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid",
-                      "type": "string"
-                    }
-                  }
-                },
-                "comments": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                  "type": "string"
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_game_end.json b/schemas/2.1/ge_game_end.json
deleted file mode 100644
index 1a9383989f..0000000000
--- a/schemas/2.1/ge_game_end.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "length",
-                "tmsize",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "tmsize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_game_update.json b/schemas/2.1/ge_game_update.json
deleted file mode 100644
index f42c239e27..0000000000
--- a/schemas/2.1/ge_game_update.json
+++ /dev/null
@@ -1,80 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "mode",
-                "ver",
-                "size",
-                "err",
-                "gid"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "mode":{
-                   "enum": [
-                    "MDATA",
-                    "WIFI",
-                    "SDCARD",
-                    "BT",
-                    "OTHER"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_genie_end.json b/schemas/2.1/ge_genie_end.json
deleted file mode 100644
index bab84d4551..0000000000
--- a/schemas/2.1/ge_genie_end.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_genie_resume.json b/schemas/2.1/ge_genie_resume.json
deleted file mode 100644
index 60b2e4cdc6..0000000000
--- a/schemas/2.1/ge_genie_resume.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_resume",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_RESUME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_genie_start.json b/schemas/2.1/ge_genie_start.json
deleted file mode 100644
index eb49966c1f..0000000000
--- a/schemas/2.1/ge_genie_start.json
+++ /dev/null
@@ -1,146 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "dspec",
-                "loc"
-              ],
-              "properties": {
-                "dspec": {
-                  "id": "http://api.ekstep.org/telemetry/edata/dspec",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "os",
-                    "make",
-                    "dname",
-                    "dlocname",
-                    "mem",
-                    "id",
-                    "idisk",
-                    "edisk",
-                    "scrn",
-                    "camera",
-                    "cpu",
-                    "sims",
-                    "cap"
-                  ],
-                  "properties": {
-                    "os": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "dname": {
-                      "type": "string"
-                    },
-                    "dlocname": {
-                      "type": "string"
-                    },
-                    "make": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "id": {
-                      "type": "string"
-                    },
-                    "idisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "edisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "mem": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "sims": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "scrn": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "camera": {
-                      "type": "string"
-                    },
-                    "cpu": {
-                      "type": "string"
-                    },
-                    "cap": {
-                      "type": "array",
-                      "items": {
-                        "type": "string"
-                      }
-                    },
-                    "mdata": {
-                      "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata",
-                      "type": "object",
-                      "additionalProperties": false,
-                      "required": [
-                        "type",
-                        "id"
-                      ],
-                      "properties": {
-                        "type": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type",
-                          "type": "string"
-                        },
-                        "id": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id",
-                          "type": "string"
-                        }
-                      }
-                    }
-                  }
-                },
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_genie_update.json b/schemas/2.1/ge_genie_update.json
deleted file mode 100644
index 7e081f0f80..0000000000
--- a/schemas/2.1/ge_genie_update.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_genie_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GENIE_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "mode": {
-                  "enum": [
-                    "MDATA",
-                    "WIFI"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                },
-                "referrer": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/referrer",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_import.json b/schemas/2.1/ge_import.json
deleted file mode 100644
index 64e7038707..0000000000
--- a/schemas/2.1/ge_import.json
+++ /dev/null
@@ -1,100 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_IMPORT"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "did": {
-              "type": "string"
-            },
-            "telemetryEvents": {
-              "type": "number"
-            },
-            "userProfiles": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_interact.json b/schemas/2.1/ge_interact.json
deleted file mode 100644
index fdc0058d0a..0000000000
--- a/schemas/2.1/ge_interact.json
+++ /dev/null
@@ -1,117 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "stageid",
-                "type",
-                "subtype",
-                "extype",
-                "pos",
-                "values",
-                "id",
-                "tid",
-                "uri"
-              ],
-              "properties": {
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                  },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                    "TOUCH",
-                    "SHOW",
-                    "HIDE",
-                    "DRAG",
-                    "DROP",
-                    "PINCH",
-                    "ZOOM",
-                    "SHAKE",
-                    "ROTATE",
-                    "SPEAK",
-                    "LISTEN",
-                    "WRITE",
-                    "DRAW",
-                    "START",
-                    "END",
-                    "CHOOSE",
-                    "ACTIVATE",
-                    "OTHER"
-                  ]
-                },
-                "subtype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                  "type": "string"
-                },
-                "extype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/extype",
-                  "type": "string"
-                },
-                "pos": {
-                  "id": "http://api.ekstep.org/telemetry/edata/pos",
-                  "type": "array"
-                },
-                "values": {
-                  "id": "http://api.ekstep.org/telemetry/edata/values",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "id": {
-                  "id": "http://api.ekstep.org/telemetry/edata/id",
-                  "type": "string"
-                },
-                "tid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/tid",
-                  "type": "string"
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/uri",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_interrupt.json b/schemas/2.1/ge_interrupt.json
deleted file mode 100644
index 175f72bf2c..0000000000
--- a/schemas/2.1/ge_interrupt.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "id"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND"
-                  ]
-                },
-               "id": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_launch_game.json b/schemas/2.1/ge_launch_game.json
deleted file mode 100644
index afdc6da457..0000000000
--- a/schemas/2.1/ge_launch_game.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_launch_game",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_LAUNCH_GAME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "tmschm",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "tmschm": {
-                  "enum": [
-                    "NONE",
-                    "INTENT",
-                    "FILE"
-                  ]
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_misc.json b/schemas/2.1/ge_misc.json
deleted file mode 100644
index ef470b6a7a..0000000000
--- a/schemas/2.1/ge_misc.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_misc",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_MISC"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_partner_data.json b/schemas/2.1/ge_partner_data.json
deleted file mode 100644
index 381e19748f..0000000000
--- a/schemas/2.1/ge_partner_data.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_PARTNER_DATA"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-       "type": "string",
-      "enum": [
-        "2.1"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            },
-            "data": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_profile_set.json b/schemas/2.1/ge_profile_set.json
deleted file mode 100644
index 9b0afad194..0000000000
--- a/schemas/2.1/ge_profile_set.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_profile_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_PROFILE_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid",
-                "attrs"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "attrs": {
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_register_partner.json b/schemas/2.1/ge_register_partner.json
deleted file mode 100644
index 859390fc69..0000000000
--- a/schemas/2.1/ge_register_partner.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_REGISTER_PARTNER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.1"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickey": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_service_api_call.json b/schemas/2.1/ge_service_api_call.json
deleted file mode 100644
index 14201f1ec0..0000000000
--- a/schemas/2.1/ge_service_api_call.json
+++ /dev/null
@@ -1,127 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_SERVICE_API_CALL"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.1"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "message": {
-              "type": "string"
-            },
-            "service": {
-              "type": "string"
-            },
-            "result": {
-              "type": "object",
-              "additionalProperties": true
-            },
-            "method": {
-              "type": "string"
-            },
-            "errorMessages": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "error": {
-              "type": "string"
-            },
-            "status": {
-              "type": "string"
-            },
-            "mode": {
-              "type": "string",
-              "enum": [
-                "WIFI",
-                "MDATA",
-                "LOCAL",
-                ""
-              ]
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_session_end.json b/schemas/2.1/ge_session_end.json
deleted file mode 100644
index 494b43163a..0000000000
--- a/schemas/2.1/ge_session_end.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-               "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_session_start.json b/schemas/2.1/ge_session_start.json
deleted file mode 100644
index 0c46fdc93e..0000000000
--- a/schemas/2.1/ge_session_start.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/ge_start_partner_session.json b/schemas/2.1/ge_start_partner_session.json
deleted file mode 100644
index 8712755d73..0000000000
--- a/schemas/2.1/ge_start_partner_session.json
+++ /dev/null
@@ -1,90 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_START_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {}
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_stop_partner_session.json b/schemas/2.1/ge_stop_partner_session.json
deleted file mode 100644
index 1b993ae8a6..0000000000
--- a/schemas/2.1/ge_stop_partner_session.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_STOP_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "length": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_transfer.json b/schemas/2.1/ge_transfer.json
deleted file mode 100644
index e9b16a75a2..0000000000
--- a/schemas/2.1/ge_transfer.json
+++ /dev/null
@@ -1,142 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_transfer",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_TRANSFER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "direction": {
-              "enum": [
-                "EXPORT",
-                "IMPORT"
-              ]
-            },
-            "datatype": {
-              "enum": [
-                "TELEMETRY",
-                "CONTENT",
-                "PROFILE",
-                "EXPLODEDCONTENT"
-              ]
-            },
-            "count": {
-              "type": "number"
-            },
-            "size": {
-              "type": "number"
-            },
-            "contents": {
-              "type": "array",
-              "items": {
-                "type": "object",
-                "additionalProperties": true,
-                "properties": {
-                  "identifier": {
-                    "type": "string"
-                  },
-                  "origin": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "transferId": {
-                    "type": "string",
-                    "minLength": 1
-                  },
-                  "transferCount": {
-                    "type": "number"
-                  },
-                  "pkgVersion": {
-                    "type": "number"
-                  }
-                }
-              }
-            }
-          },
-          "required": [
-            "direction",
-            "datatype",
-            "count"
-          ]
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_update_profile.json b/schemas/2.1/ge_update_profile.json
deleted file mode 100644
index 0f4b713aaa..0000000000
--- a/schemas/2.1/ge_update_profile.json
+++ /dev/null
@@ -1,115 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_UPDATE_PROFILE"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "minLength": 1
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "uid": {
-              "type": "string"
-            },
-            "handle": {
-              "type": "string"
-            },
-            "gender": {
-              "type": "string"
-            },
-            "age": {
-              "type": "number"
-            },
-            "standard": {
-              "type": "number"
-            },
-            "language": {
-              "type": "string"
-            },
-            "day": {
-              "type": "number"
-            },
-            "month": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.1/ge_view_progress.json b/schemas/2.1/ge_view_progress.json
deleted file mode 100644
index 25e05c7d48..0000000000
--- a/schemas/2.1/ge_view_progress.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_view_progress",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_VIEW_PROGRESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "ueksid"
-              ],
-              "properties": {
-                "ueksid": {
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_assess.json b/schemas/2.1/oe_assess.json
deleted file mode 100644
index b5b21c07ba..0000000000
--- a/schemas/2.1/oe_assess.json
+++ /dev/null
@@ -1,136 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/oe_assess",
-  "type": "object",
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ASSESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "qid",
-                "pass",
-                "score",
-                "resvalues",
-                "length",
-                "exlength",
-                "params",
-                "uri",
-                "qindex"
-              ],
-              "properties": {
-                "qid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qid",
-                  "type": "string"
-                },
-                "pass": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/pass",
-                  "enum": [
-                    "Yes",
-                    "No"
-                  ]
-                },
-                "score": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/score",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                },
-                "length": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/length",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "exlength": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/exlength",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "params": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/params",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/uri",
-                  "type": "string"
-                },
-                "qindex": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qindex",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "qtitle": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                  "type": "string"
-                },
-                "qdesc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                  "type": "string"
-                },
-                "maxscore": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "mc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                },
-                "mmc": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mmc",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ],
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata",
-    "pdata",
-    "channel"
-  ]
-}
diff --git a/schemas/2.1/oe_earn.json b/schemas/2.1/oe_earn.json
deleted file mode 100644
index d4dd750d55..0000000000
--- a/schemas/2.1/oe_earn.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_earn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_EARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "required": [
-                "type",
-                "points"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                     "MONEY","GEMS","POINTS"
-                  ]
-                },
-                "points": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_end.json b/schemas/2.1/oe_end.json
deleted file mode 100644
index e9d99c1747..0000000000
--- a/schemas/2.1/oe_end.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length":{
-                  "type": "number",
-                  "minimum": 0
-                },
-                "progress":{
-                  "type": "number",
-                  "minimum": 0
-                },
-                "stageid":{
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_error.json b/schemas/2.1/oe_error.json
deleted file mode 100644
index e21cd17893..0000000000
--- a/schemas/2.1/oe_error.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_error",
-  "type": "object",
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "env",
-                "type",
-                "stageid",
-                "objecttype",
-                "objectid",
-                "err",
-                "action",
-                "data",
-                "severity"
-              ],
-              "properties": {
-                "env": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/env",
-                  "type": "string"
-                },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "enum": [
-                    "SYSTEM",
-                    "API",
-                    "PLUGIN",
-                    "CONTENT",
-                    "ASSET",
-                    "OTHER"
-                  ]
-                },
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/stageid",
-                  "type": "string"
-                },
-                "objecttype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/objecttype",
-                  "type": "string"
-                },
-                "objectid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/objectid",
-                  "type": "string"
-                },
-                "err": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/err",
-                  "type": "string"
-                },
-                "action": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/action",
-                  "type": "string"
-                },
-                "data": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/data",
-                  "type": "string"
-                },
-                "severity": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/severity",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ],
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "edata",
-    "pdata",
-    "channel"
-  ]
-}
diff --git a/schemas/2.1/oe_feedback.json b/schemas/2.1/oe_feedback.json
deleted file mode 100644
index bb19958ed8..0000000000
--- a/schemas/2.1/oe_feedback.json
+++ /dev/null
@@ -1,82 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "rating",
-                "context",
-                "comments",
-                "resvalues"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                  "type": "string",
-                  "enum": [
-                    "FLAG", "SURVEY", "RATING"
-                  ]
-                },
-                "rating": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                  "type": "number"
-                },
-                "context": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                  "type": "string"
-                },
-                "comments": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                  "type": "string"
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_interact.json b/schemas/2.1/oe_interact.json
deleted file mode 100644
index e6faba5fa2..0000000000
--- a/schemas/2.1/oe_interact.json
+++ /dev/null
@@ -1,121 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "stageid",
-                "type",
-                "subtype",
-                "extype",
-                "pos",
-                "values",
-                "id",
-                "tid",
-                "uri"
-              ],
-              "properties": {
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                  },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                    "TOUCH",
-                    "SHOW",
-                    "HIDE",
-                    "DRAG",
-                    "DROP",
-                    "PINCH",
-                    "ZOOM",
-                    "SHAKE",
-                    "ROTATE",
-                    "SPEAK",
-                    "LISTEN",
-                    "WRITE",
-                    "DRAW",
-                    "START",
-                    "END",
-                    "CHOOSE",
-                    "ACTIVATE",
-                    "OTHER",
-                    "SCROLL",
-                    "HEARTBEAT"
-                  ]
-                },
-                "subtype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                  "type": "string"
-                },
-                "extype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/extype",
-                  "type": "string"
-                },
-                "pos": {
-                  "id": "http://api.ekstep.org/telemetry/edata/pos",
-                  "type": "array"
-                },
-                "values": {
-                  "id": "http://api.ekstep.org/telemetry/edata/values",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                },
-                "id": {
-                  "id": "http://api.ekstep.org/telemetry/edata/id",
-                  "type": "string"
-                },
-                "tid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/tid",
-                  "type": "string"
-                },
-                "uri": {
-                  "id": "http://api.ekstep.org/telemetry/edata/uri",
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_interrupt.json b/schemas/2.1/oe_interrupt.json
deleted file mode 100644
index 048ce25afa..0000000000
--- a/schemas/2.1/oe_interrupt.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "stageid"
-              ],
-              "properties": {
-                "type": {
-                  "type": "string",
-                  "enum": [
-                    "IDLE","SLEEP","CALL","SWITCH","LOCK","OTHER","BACKGROUND","RESUME"
-                  ]
-                },
-               "stageid": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_item_response.json b/schemas/2.1/oe_item_response.json
deleted file mode 100644
index 426a608495..0000000000
--- a/schemas/2.1/oe_item_response.json
+++ /dev/null
@@ -1,81 +0,0 @@
-
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_ITEM_RESPONSE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "qid",
-                "type",
-                "state",
-                "resvalues"
-              ],
-              "properties": {
-                "qid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/qid",
-                  "type": "string"
-                },
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string",
-                  "enum": [
-                     "CHOOSE","DRAG","SELECT","MATCH","INPUT","SPEAK","WRITE"
-                  ]
-                },
-                "state": {
-                  "id": "http://api.ekstep.org/telemetry/edata/state",
-                  "type": "string",
-                  "enum": [
-                    "SELECTED", "UNSELECTED"
-                  ]
-                },
-                "resvalues": {
-                  "id": "http://api.ekstep.org/telemetry/edata/res",
-                  "type": "array",
-                    "items": {
-                      "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_learn.json b/schemas/2.1/oe_learn.json
deleted file mode 100644
index d92668ac28..0000000000
--- a/schemas/2.1/oe_learn.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_learn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEARN"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "topics"
-              ],
-              "properties": {
-                "topics": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/topics",
-                  "type": "array",
-                  "minItems": 1,
-                  "items": {
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                      "mc",
-                      "methods"
-                    ],
-                    "properties": {
-                      "mc": {
-                        "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                        "type": "string"
-                      },
-                      "methods": {
-                        "id": "http://api.ekstep.org/telemetry/edata/eks/methods",
-                        "type": "array",
-                        "minItems": 1,
-                        "items": {
-                          "type": "string",
-                          "enum": [
-                            "PLAY",
-                            "ANSWER",
-                            "WRITE",
-                            "SPEAK",
-                            "OTHER"
-                          ]
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_level_set.json b/schemas/2.1/oe_level_set.json
deleted file mode 100644
index ef160fa029..0000000000
--- a/schemas/2.1/oe_level_set.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_level_set",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_LEVEL_SET"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "category",
-                "current",
-                "max"
-              ],
-              "properties": {
-                "category": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/category",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "current": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/current",
-                  "type": "string",
-                  "minLength": 1
-                },
-                "max": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/max",
-                  "type": "string",
-                  "minLength": 1
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_navigate.json b/schemas/2.1/oe_navigate.json
deleted file mode 100644
index ccfdeb8960..0000000000
--- a/schemas/2.1/oe_navigate.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_NAVIGATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "type",
-                "itype",
-                "stageid",
-                "stageto"
-              ],
-              "properties": {
-                "type": {
-                  "id": "http://api.ekstep.org/telemetry/edata/type",
-                  "type": "string"
-                },
-                "itype": {
-                  "id": "http://api.ekstep.org/telemetry/edata/itype",
-                  "type": "string"
-                },
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                  "type": "string"
-                },
-                "stageto": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_start.json b/schemas/2.1/oe_start.json
deleted file mode 100644
index 461743157c..0000000000
--- a/schemas/2.1/oe_start.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-           "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "mode": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/mode",  
-                  "type": "string"
-                },
-                "stageid": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/stageid",  
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_summary.json b/schemas/2.1/oe_summary.json
deleted file mode 100644
index 77349bc296..0000000000
--- a/schemas/2.1/oe_summary.json
+++ /dev/null
@@ -1,210 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_earn",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_SUMMARY"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "required": [
-                "timespent"
-              ],
-              "properties": {
-                "timespent": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/timespent",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "interrupts": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/interrupts",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "screensummary": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary",
-                  "type": "array",
-                  "items": {
-                      "type": "object"
-                  },
-                  "properties": {
-                    "id": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/id",
-                      "type": "string"
-                    },
-                    "timespent": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/timespent",
-                      "type": "number",
-                      "minimum": 0
-                    },
-                    "count": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/count",
-                      "type": "number",
-                      "minimum": 0
-                    }
-                  }
-                },
-                "eventsummary": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary",
-                  "type": "array",
-                  "items": {
-                      "type": "object"
-                  },
-                  "properties": {
-                    "id": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/id",
-                      "type": "string"
-                    },
-                    "count": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/count",
-                      "type": "number",
-                      "minimum": 0
-                    }
-                  }
-                },
-                "items":{
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/items",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  },
-                  "properties":{
-                    "qid": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/qid",
-                      "type" : "string"
-                    },
-                     "pass": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/pass",
-                      "type" : "string"
-                    },
-                     "maxscore": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/maxscore",
-                      "type" : "number",
-                      "minimum": 0
-                    },
-                     "score": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/score",
-                      "type" : "string"
-                    },
-                     "resvalues": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/resvalues",
-                      "type" : "array",
-                      "items": {
-                        "type": "object"
-                      }
-                    },
-                    "length": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/length",
-                      "type" : "number",
-                      "minimum": 0
-                    },
-                    "qindex" : {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/qindex",
-                      "type" : "number",
-                      "minimum": 0
-                    },
-                    "qdesc": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/qdesc",
-                      "type" : "string"
-                    },
-                    "qtitle": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/qtitle",
-                      "type" : "string"
-                    },
-                    "mmc": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/mmc",
-                      "type" : "array",
-                       "items": {
-                        "type": "string"
-                      }
-                    },
-                    "mc": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/items/mc",
-                      "type" : "array",
-                       "items": {
-                        "type": "string"
-                      }
-                    }
-                  }
-                },
-                "itemsummary": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary",
-                  "type": "array",
-                  "items": {
-                      "type": "object"
-                  },
-                  "properties": {
-                    "qid": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/qid",
-                      "type": "string"
-                    },
-                    "score": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/score",
-                      "type": "number",
-                      "minimum": 0
-                    },
-                    "attempts": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/attempts",
-                      "type": "number",
-                      "minimum": 0
-                    },
-                     "timespent": {
-                      "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/timespent",
-                      "type": "number",
-                      "minimum": 0
-                    }
-                  }
-                },
-                "interactevents": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/interactevents",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "interacteventspermin": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/interacteventspermin",
-                  "type": "number",
-                  "minimum": 0
-                },
-                "progress": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/progress",
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/oe_xapi.json b/schemas/2.1/oe_xapi.json
deleted file mode 100644
index 4652d9bab6..0000000000
--- a/schemas/2.1/oe_xapi.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/oe_xapi",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.1/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "OE_XAPI"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "xapi"
-              ],
-              "properties": {
-                "xapi": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/xapi",
-                  "type": "object"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.1/pdata.json b/schemas/2.1/pdata.json
deleted file mode 100644
index cd509bc58b..0000000000
--- a/schemas/2.1/pdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/pdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/pdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/2.2/cdata.json b/schemas/2.2/cdata.json
deleted file mode 100644
index 4c7261b025..0000000000
--- a/schemas/2.2/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
diff --git a/schemas/2.2/common.json b/schemas/2.2/common.json
deleted file mode 100644
index 8c311a231e..0000000000
--- a/schemas/2.2/common.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/gdata.json"
-    },
-    "pdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/pdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 1
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 1
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "http://localhost:7070/schemas/2.2/etags.json"
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/2.2/common_empty_sid_uid.json b/schemas/2.2/common_empty_sid_uid.json
deleted file mode 100644
index 6ba8549c7e..0000000000
--- a/schemas/2.2/common_empty_sid_uid.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "pdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/pdata.json"
-    },
-    "gdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/gdata.json"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "etags": {
-      "$ref": "http://localhost:7070/schemas/2.2/etags.json"
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/2.2/cdata.json"
-    },
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/2.2/channel",
-      "type": "string",
-      "minLength": 1
-    }
-  }
-}
diff --git a/schemas/2.2/etags.json b/schemas/2.2/etags.json
deleted file mode 100644
index e00c02e041..0000000000
--- a/schemas/2.2/etags.json
+++ /dev/null
@@ -1,30 +0,0 @@
-{
-  "type": "object",
-    "properties": {
-      "app": {
-        "id": "http://api.ekstep.org/telemetry/etags/app",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/app/items",
-            "type": "string"
-        }
-      },
-      "partner": {
-        "id": "http://api.ekstep.org/telemetry/etags/partner",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/partner/items",
-            "type": "string"
-        }
-      },
-      "dims": {
-        "id": "http://api.ekstep.org/telemetry/etags/dims",
-        "type": "array",
-        "items": {
-            "id": "http://api.ekstep.org/telemetry/etags/dims/items",
-            "type": "string"
-        }
-      }
-    },
-    "additionalProperties": false
-}
diff --git a/schemas/2.2/event.json b/schemas/2.2/event.json
deleted file mode 100644
index a63c71e16e..0000000000
--- a/schemas/2.2/event.json
+++ /dev/null
@@ -1,126 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [{
-      "$ref": "http://localhost:7070/schemas/2.2/ge_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_resume.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_service_api_call.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_session_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_session_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_game_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_profile_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_partner_data.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_register_partner.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_start_partner_session.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_stop_partner_session.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_view_progress.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_game_update.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_assess.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_level_set.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_learn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_earn.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_item_response.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_navigate.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_create_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_update_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_transfer.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_delete_profile.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_summary.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/oe_xapi.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_create_user.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/ge_launch_game.json"
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/gdata.json b/schemas/2.2/gdata.json
deleted file mode 100644
index 1b603a1b7b..0000000000
--- a/schemas/2.2/gdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/2.2/ge_api_call.json b/schemas/2.2/ge_api_call.json
deleted file mode 100644
index 2de5d7f6c3..0000000000
--- a/schemas/2.2/ge_api_call.json
+++ /dev/null
@@ -1,80 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_api_call",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_API_CALL"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "api",
-                "msgid",
-                "isize",
-                "osize",
-                "length",
-                "err"
-              ],
-              "properties": {
-                "api": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "msgid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "isize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "osize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_create_profile.json b/schemas/2.2/ge_create_profile.json
deleted file mode 100644
index 0b93ef9987..0000000000
--- a/schemas/2.2/ge_create_profile.json
+++ /dev/null
@@ -1,126 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "properties": {
-        "eid": {
-            "id": "http://api.ekstep.org/telemetry/eid",
-            "enum": [
-                "GE_CREATE_PROFILE"
-            ]
-        },
-        "ets": {
-            "id": "http://api.ekstep.org/telemetry/ets",
-            "type": "number",
-            "format": "date-time"
-        },
-        "ver": {
-            "id": "http://api.ekstep.org/telemetry/ver",
-            "type": "string",
-            "enum": [
-                "2.2"
-            ]
-        },
-        "gdata": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/id",
-                    "type": "string"
-                },
-                "ver": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/ver",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": [
-                "id",
-                "ver"
-            ]
-        },
-        "sid": {
-            "id": "http://api.ekstep.org/telemetry/sid",
-            "type": "string",
-            "minLength": 0
-        },
-        "uid": {
-            "id": "http://api.ekstep.org/telemetry/uid",
-            "type": "string",
-            "minLength": 0
-        },
-        "did": {
-            "id": "http://api.ekstep.org/telemetry/did",
-            "type": "string",
-            "minLength": 1
-        },
-        "tags": {
-            "type": "array",
-            "items": {
-                "type": "object"
-            }
-        },
-        "edata": {
-            "id": "http://api.ekstep.org/telemetry/edata",
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-                "eks"
-            ],
-            "properties": {
-                "eks": {
-                    "id": "http://api.ekstep.org/telemetry/edata/eks",
-                    "type": "object",
-                    "additionalProperties": true,
-                    "properties": {
-                        "loc": {
-                            "type": "string"
-                        },
-                        "uid": {
-                            "type": "string"
-                        },
-                        "handle": {
-                            "type": "string"
-                        },
-                        "gender": {
-                            "type": "string"
-                        },
-                        "age": {
-                            "type": "number"
-                        },
-                        "standard": {
-                            "type": "number"
-                        },
-                        "language": {
-                            "type": "string"
-                        },
-                        "day": {
-                            "type": "number"
-                        },
-                        "month": {
-                            "type": "number"
-                        },
-                        "board": {
-                            "type": "string"
-                        },
-                        "medium": {
-                            "type": "string"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_create_user.json b/schemas/2.2/ge_create_user.json
deleted file mode 100644
index 952f537aa8..0000000000
--- a/schemas/2.2/ge_create_user.json
+++ /dev/null
@@ -1,99 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "properties": {
-        "eid": {
-            "id": "http://api.ekstep.org/telemetry/eid",
-            "enum": [
-                "GE_CREATE_USER"
-            ]
-        },
-        "ets": {
-            "id": "http://api.ekstep.org/telemetry/ets",
-            "type": "number",
-            "format": "date-time"
-        },
-        "ver": {
-            "id": "http://api.ekstep.org/telemetry/ver",
-            "type": "string",
-            "enum": [
-                "2.2"
-            ]
-        },
-        "gdata": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/id",
-                    "type": "string"
-                },
-                "ver": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/ver",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": [
-                "id",
-                "ver"
-            ]
-        },
-        "sid": {
-            "id": "http://api.ekstep.org/telemetry/sid",
-            "type": "string",
-            "minLength": 0
-        },
-        "uid": {
-            "id": "http://api.ekstep.org/telemetry/uid",
-            "type": "string",
-            "minLength": 0
-        },
-        "did": {
-            "id": "http://api.ekstep.org/telemetry/did",
-            "type": "string",
-            "minLength": 1
-        },
-        "tags": {
-            "type": "array",
-            "items": {
-                "type": "object"
-            }
-        },
-        "edata": {
-            "id": "http://api.ekstep.org/telemetry/edata",
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-                "eks"
-            ],
-            "properties": {
-                "eks": {
-                    "id": "http://api.ekstep.org/telemetry/edata/eks",
-                    "type": "object",
-                    "additionalProperties": true,
-                    "properties": {
-                        "loc": {
-                            "type": "string"
-                        },
-                        "uid": {
-                            "type": "string"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_delete_profile.json b/schemas/2.2/ge_delete_profile.json
deleted file mode 100644
index a87c0ea40b..0000000000
--- a/schemas/2.2/ge_delete_profile.json
+++ /dev/null
@@ -1,99 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_delete_profile",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "properties": {
-        "eid": {
-            "id": "http://api.ekstep.org/telemetry/eid",
-            "enum": [
-                "GE_DELETE_PROFILE"
-            ]
-        },
-        "ets": {
-            "id": "http://api.ekstep.org/telemetry/ets",
-            "type": "number",
-            "format": "date-time"
-        },
-        "ver": {
-            "id": "http://api.ekstep.org/telemetry/ver",
-            "type": "string",
-            "enum":[
-                "2.2"
-            ]
-        },
-        "gdata": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/id",
-                    "type": "string"
-                },
-                "ver": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/ver",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": [
-                "id",
-                "ver"
-            ]
-        },
-        "sid": {
-            "id": "http://api.ekstep.org/telemetry/sid",
-            "type": "string",
-            "minLength": 0
-        },
-        "uid": {
-            "id": "http://api.ekstep.org/telemetry/uid",
-            "type": "string",
-            "minLength": 0
-        },
-        "did": {
-            "id": "http://api.ekstep.org/telemetry/did",
-            "type": "string",
-            "minLength": 1
-        },
-        "tags": {
-            "type": "array",
-            "items": {
-                "type": "object"
-            }
-        },
-        "edata": {
-            "id": "http://api.ekstep.org/telemetry/edata",
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-                "eks"
-            ],
-            "properties": {
-                "eks": {
-                    "id": "http://api.ekstep.org/telemetry/edata/eks",
-                    "type": "object",
-                    "additionalProperties": true,
-                    "properties": {
-                        "uid": {
-                            "type": "string"
-                        },
-                        "duration": {
-                            "type": "number"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_end.json b/schemas/2.2/ge_end.json
deleted file mode 100644
index 9f0a942821..0000000000
--- a/schemas/2.2/ge_end.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_error.json b/schemas/2.2/ge_error.json
deleted file mode 100644
index 646a22e29a..0000000000
--- a/schemas/2.2/ge_error.json
+++ /dev/null
@@ -1,77 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_ERROR"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "id",
-                                "err",
-                                "eventId",
-                                "data"
-                            ],
-                            "properties": {
-                                "err": {
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "SYSTEM",
-                                        "GENIE",
-                                        "GAME",
-                                        "GENIESERVICES"
-                                    ]
-                                },
-                                "id": {
-                                    "type": "string"
-                                },
-                                "eventId": {
-                                    "type": "string"
-                                },
-                                "data": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_feedback.json b/schemas/2.2/ge_feedback.json
deleted file mode 100644
index 068273a041..0000000000
--- a/schemas/2.2/ge_feedback.json
+++ /dev/null
@@ -1,103 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_end",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_FEEDBACK"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "rating",
-                                "context",
-                                "comments"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                                    "type": "string",
-                                    "enum": [
-                                        "FLAG",
-                                        "SURVEY",
-                                        "RATING"
-                                    ]
-                                },
-                                "rating": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                                    "type": "number"
-                                },
-                                "context": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                                    "type": "object",
-                                    "additionalProperties": false,
-                                    "required": [
-                                        "type",
-                                        "id",
-                                        "stageid"
-                                    ],
-                                    "properties": {
-                                        "type": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/context/type",
-                                            "type": "string"
-                                        },
-                                        "id": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/context/id",
-                                            "type": "string"
-                                        },
-                                        "stageid": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/context/stageid",
-                                            "type": "string"
-                                        }
-                                    }
-                                },
-                                "comments": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                                    "type": "string"
-                                },
-                                "resvalues": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/res",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_game_end.json b/schemas/2.2/ge_game_end.json
deleted file mode 100644
index d7f6d30243..0000000000
--- a/schemas/2.2/ge_game_end.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_game_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_GAME_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "gid",
-                "length",
-                "tmsize",
-                "err"
-              ],
-              "properties": {
-                "gid": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "length": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "tmsize": {
-                  "type": "number",
-                  "minimum": 0
-                },
-                "err": {
-                   "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_game_update.json b/schemas/2.2/ge_game_update.json
deleted file mode 100644
index 57428bb22c..0000000000
--- a/schemas/2.2/ge_game_update.json
+++ /dev/null
@@ -1,80 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_game_update",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_GAME_UPDATE"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "mode",
-                                "ver",
-                                "size",
-                                "err",
-                                "gid"
-                            ],
-                            "properties": {
-                                "gid": {
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "mode": {
-                                    "enum": [
-                                        "MDATA",
-                                        "WIFI",
-                                        "SDCARD",
-                                        "BT",
-                                        "OTHER"
-                                    ]
-                                },
-                                "ver": {
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "size": {
-                                    "type": "number",
-                                    "minimum": -1
-                                },
-                                "err": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_interact.json b/schemas/2.2/ge_interact.json
deleted file mode 100644
index ac2f99645d..0000000000
--- a/schemas/2.2/ge_interact.json
+++ /dev/null
@@ -1,117 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_interact",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_INTERACT"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "stageid",
-                                "type",
-                                "subtype",
-                                "extype",
-                                "pos",
-                                "values",
-                                "id",
-                                "tid",
-                                "uri"
-                            ],
-                            "properties": {
-                                "stageid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/type",
-                                    "type": "string",
-                                    "enum": [
-                                        "TOUCH",
-                                        "SHOW",
-                                        "HIDE",
-                                        "DRAG",
-                                        "DROP",
-                                        "PINCH",
-                                        "ZOOM",
-                                        "SHAKE",
-                                        "ROTATE",
-                                        "SPEAK",
-                                        "LISTEN",
-                                        "WRITE",
-                                        "DRAW",
-                                        "START",
-                                        "END",
-                                        "CHOOSE",
-                                        "ACTIVATE",
-                                        "OTHER"
-                                    ]
-                                },
-                                "subtype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                                    "type": "string"
-                                },
-                                "extype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/extype",
-                                    "type": "string"
-                                },
-                                "pos": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/pos",
-                                    "type": "array"
-                                },
-                                "values": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/values",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                },
-                                "id": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/id",
-                                    "type": "string"
-                                },
-                                "tid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/tid",
-                                    "type": "string"
-                                },
-                                "uri": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/uri",
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_interrupt.json b/schemas/2.2/ge_interrupt.json
deleted file mode 100644
index 3638e71ebb..0000000000
--- a/schemas/2.2/ge_interrupt.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_interrupt",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_INTERRUPT"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "id"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "IDLE",
-                                        "SLEEP",
-                                        "CALL",
-                                        "SWITCH",
-                                        "LOCK",
-                                        "OTHER",
-                                        "BACKGROUND"
-                                    ]
-                                },
-                                "id": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_launch_game.json b/schemas/2.2/ge_launch_game.json
deleted file mode 100644
index fc7247655b..0000000000
--- a/schemas/2.2/ge_launch_game.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_launch_game",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_LAUNCH_GAME"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "gid",
-                                "tmschm",
-                                "err"
-                            ],
-                            "properties": {
-                                "gid": {
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "tmschm": {
-                                    "enum": [
-                                        "NONE",
-                                        "INTENT",
-                                        "FILE"
-                                    ]
-                                },
-                                "err": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_partner_data.json b/schemas/2.2/ge_partner_data.json
deleted file mode 100644
index d0e26c4175..0000000000
--- a/schemas/2.2/ge_partner_data.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_PARTNER_DATA"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            },
-            "data": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.2/ge_profile_set.json b/schemas/2.2/ge_profile_set.json
deleted file mode 100644
index 5461c28c28..0000000000
--- a/schemas/2.2/ge_profile_set.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_profile_set",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_PROFILE_SET"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "ueksid",
-                                "attrs"
-                            ],
-                            "properties": {
-                                "ueksid": {
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "attrs": {
-                                    "type": "array",
-                                    "items": {
-                                        "type": "string"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_register_partner.json b/schemas/2.2/ge_register_partner.json
deleted file mode 100644
index 9c2e9919d7..0000000000
--- a/schemas/2.2/ge_register_partner.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_REGISTER_PARTNER"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "partnerid": {
-              "type": "string"
-            },
-            "publickey": {
-              "type": "string"
-            },
-            "publickeyid": {
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.2/ge_resume.json b/schemas/2.2/ge_resume.json
deleted file mode 100644
index c259d3f252..0000000000
--- a/schemas/2.2/ge_resume.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_resume",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_RESUME"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_service_api_call.json b/schemas/2.2/ge_service_api_call.json
deleted file mode 100644
index 7c34d134b6..0000000000
--- a/schemas/2.2/ge_service_api_call.json
+++ /dev/null
@@ -1,127 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_SERVICE_API_CALL"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "message": {
-              "type": "string"
-            },
-            "service": {
-              "type": "string"
-            },
-            "result": {
-              "type": "object",
-              "additionalProperties": true
-            },
-            "method": {
-              "type": "string"
-            },
-            "errorMessages": {
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "error": {
-              "type": "string"
-            },
-            "status": {
-              "type": "string"
-            },
-            "mode": {
-              "type": "string",
-              "enum": [
-                "WIFI",
-                "MDATA",
-                "LOCAL",
-                ""
-              ]
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.2/ge_session_end.json b/schemas/2.2/ge_session_end.json
deleted file mode 100644
index b2238ff328..0000000000
--- a/schemas/2.2/ge_session_end.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "length"
-              ],
-              "properties": {
-               "length": {
-                  "type": "number",
-                  "minimum": 0
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_session_start.json b/schemas/2.2/ge_session_start.json
deleted file mode 100644
index 085f3f7779..0000000000
--- a/schemas/2.2/ge_session_start.json
+++ /dev/null
@@ -1,55 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_session_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_SESSION_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "loc"
-              ],
-              "properties": {
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_start.json b/schemas/2.2/ge_start.json
deleted file mode 100644
index b81a6bcd9b..0000000000
--- a/schemas/2.2/ge_start.json
+++ /dev/null
@@ -1,146 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "required": [
-                "dspec",
-                "loc"
-              ],
-              "properties": {
-                "dspec": {
-                  "id": "http://api.ekstep.org/telemetry/edata/dspec",
-                  "type": "object",
-                  "additionalProperties": false,
-                  "required": [
-                    "os",
-                    "make",
-                    "dname",
-                    "dlocname",
-                    "mem",
-                    "id",
-                    "idisk",
-                    "edisk",
-                    "scrn",
-                    "camera",
-                    "cpu",
-                    "sims",
-                    "cap"
-                  ],
-                  "properties": {
-                    "os": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "dname": {
-                      "type": "string"
-                    },
-                    "dlocname": {
-                      "type": "string"
-                    },
-                    "make": {
-                      "type": "string",
-                      "minLength": 1
-                    },
-                    "id": {
-                      "type": "string"
-                    },
-                    "idisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "edisk": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "mem": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "sims": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "scrn": {
-                      "type": "number",
-                      "minimum": -1
-                    },
-                    "camera": {
-                      "type": "string"
-                    },
-                    "cpu": {
-                      "type": "string"
-                    },
-                    "cap": {
-                      "type": "array",
-                      "items": {
-                        "type": "string"
-                      }
-                    },
-                    "mdata": {
-                      "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata",
-                      "type": "object",
-                      "additionalProperties": false,
-                      "required": [
-                        "type",
-                        "id"
-                      ],
-                      "properties": {
-                        "type": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/type",
-                          "type": "string"
-                        },
-                        "id": {
-                          "id": "http://api.ekstep.org/telemetry/edata/dspec/mdata/id",
-                          "type": "string"
-                        }
-                      }
-                    }
-                  }
-                },
-                "loc": {
-                  "type": "string"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_start_partner_session.json b/schemas/2.2/ge_start_partner_session.json
deleted file mode 100644
index a73b128e73..0000000000
--- a/schemas/2.2/ge_start_partner_session.json
+++ /dev/null
@@ -1,92 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_START_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {}
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.2/ge_stop_partner_session.json b/schemas/2.2/ge_stop_partner_session.json
deleted file mode 100644
index 9830bab8cd..0000000000
--- a/schemas/2.2/ge_stop_partner_session.json
+++ /dev/null
@@ -1,96 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "properties": {
-    "eid": {
-      "id": "http://api.ekstep.org/telemetry/eid",
-      "enum": [
-        "GE_STOP_PARTNER_SESSION"
-      ]
-    },
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "2.2"
-      ]
-    },
-    "gdata": {
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/gdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/gdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "ver"
-      ]
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/sid",
-      "type": "string",
-      "minLength": 0
-    },
-    "uid": {
-      "id": "http://api.ekstep.org/telemetry/uid",
-      "type": "string",
-      "minLength": 0
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/did",
-      "type": "string",
-      "minLength": 1
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "edata": {
-      "id": "http://api.ekstep.org/telemetry/edata",
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "eks"
-      ],
-      "properties": {
-        "eks": {
-          "id": "http://api.ekstep.org/telemetry/edata/eks",
-          "type": "object",
-          "additionalProperties": true,
-          "properties": {
-            "length": {
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  }
-}
diff --git a/schemas/2.2/ge_transfer.json b/schemas/2.2/ge_transfer.json
deleted file mode 100644
index a3d7d6008e..0000000000
--- a/schemas/2.2/ge_transfer.json
+++ /dev/null
@@ -1,144 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_transfer",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "properties": {
-        "eid": {
-            "id": "http://api.ekstep.org/telemetry/eid",
-            "enum": [
-                "GE_TRANSFER"
-            ]
-        },
-        "ets": {
-            "id": "http://api.ekstep.org/telemetry/ets",
-            "type": "number",
-            "format": "date-time"
-        },
-        "ver": {
-            "id": "http://api.ekstep.org/telemetry/ver",
-            "type": "string",
-            "enum": [
-                "2.2"
-            ]
-        },
-        "gdata": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/id",
-                    "type": "string"
-                },
-                "ver": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/ver",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": [
-                "id",
-                "ver"
-            ]
-        },
-        "sid": {
-            "id": "http://api.ekstep.org/telemetry/sid",
-            "type": "string",
-            "minLength": 0
-        },
-        "uid": {
-            "id": "http://api.ekstep.org/telemetry/uid",
-            "type": "string",
-            "minLength": 0
-        },
-        "did": {
-            "id": "http://api.ekstep.org/telemetry/did",
-            "type": "string",
-            "minLength": 1
-        },
-        "tags": {
-            "type": "array",
-            "items": {
-                "type": "object"
-            }
-        },
-        "edata": {
-            "id": "http://api.ekstep.org/telemetry/edata",
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-                "eks"
-            ],
-            "properties": {
-                "eks": {
-                    "id": "http://api.ekstep.org/telemetry/edata/eks",
-                    "type": "object",
-                    "additionalProperties": true,
-                    "properties": {
-                        "direction": {
-                            "enum": [
-                                "EXPORT",
-                                "IMPORT"
-                            ]
-                        },
-                        "datatype": {
-                            "enum": [
-                                "TELEMETRY",
-                                "CONTENT",
-                                "PROFILE",
-                                "EXPLODEDCONTENT"
-                            ]
-                        },
-                        "count": {
-                            "type": "number"
-                        },
-                        "size": {
-                            "type": "number"
-                        },
-                        "contents": {
-                            "type": "array",
-                            "items": {
-                                "type": "object",
-                                "additionalProperties": true,
-                                "properties": {
-                                    "identifier": {
-                                        "type": "string"
-                                    },
-                                    "origin": {
-                                        "type": "string",
-                                        "minLength": 1
-                                    },
-                                    "transferId": {
-                                        "type": "string",
-                                        "minLength": 1
-                                    },
-                                    "transferCount": {
-                                        "type": "number"
-                                    },
-                                    "pkgVersion": {
-                                        "type": "number"
-                                    }
-                                }
-                            }
-                        }
-                    },
-                    "required": [
-                        "direction",
-                        "datatype",
-                        "count"
-                    ]
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_update.json b/schemas/2.2/ge_update.json
deleted file mode 100644
index 75a0050dbc..0000000000
--- a/schemas/2.2/ge_update.json
+++ /dev/null
@@ -1,73 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/ge_update",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "gdata",
-    "sid",
-    "uid",
-    "did",
-    "mid",
-    "edata",
-    "pdata",
-    "channel"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/2.2/common_empty_sid_uid.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "GE_UPDATE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "eks"
-          ],
-          "properties": {
-            "eks": {
-              "id": "http://api.ekstep.org/telemetry/edata/eks",
-              "type": "object",
-              "additionalProperties": false,
-              "properties": {
-                "mode": {
-                  "enum": [
-                    "MDATA",
-                    "WIFI"
-                  ]
-                },
-                "ver": {
-                  "type": "string",
-                  "minLength": 1
-                },
-                "size": {
-                  "type": "number",
-                  "minimum": -1
-                },
-                "err": {
-                  "type": "string"
-                },
-                "referrer": {
-                  "id": "http://api.ekstep.org/telemetry/edata/eks/referrer",
-                  "type": "array",
-                  "items": {
-                    "type": "object"
-                  }
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
diff --git a/schemas/2.2/ge_update_profile.json b/schemas/2.2/ge_update_profile.json
deleted file mode 100644
index 7db13c06c2..0000000000
--- a/schemas/2.2/ge_update_profile.json
+++ /dev/null
@@ -1,117 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_error",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "properties": {
-        "eid": {
-            "id": "http://api.ekstep.org/telemetry/eid",
-            "enum": [
-                "GE_UPDATE_PROFILE"
-            ]
-        },
-        "ets": {
-            "id": "http://api.ekstep.org/telemetry/ets",
-            "type": "number",
-            "format": "date-time"
-        },
-        "ver": {
-            "id": "http://api.ekstep.org/telemetry/ver",
-            "type": "string",
-            "enum": [
-                "2.2"
-            ]
-        },
-        "gdata": {
-            "type": "object",
-            "properties": {
-                "id": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/id",
-                    "type": "string"
-                },
-                "ver": {
-                    "id": "http://api.ekstep.org/telemetry/gdata/ver",
-                    "type": "string"
-                }
-            },
-            "additionalProperties": false,
-            "required": [
-                "id",
-                "ver"
-            ]
-        },
-        "sid": {
-            "id": "http://api.ekstep.org/telemetry/sid",
-            "type": "string",
-            "minLength": 0
-        },
-        "uid": {
-            "id": "http://api.ekstep.org/telemetry/uid",
-            "type": "string",
-            "minLength": 0
-        },
-        "did": {
-            "id": "http://api.ekstep.org/telemetry/did",
-            "type": "string",
-            "minLength": 1
-        },
-        "tags": {
-            "type": "array",
-            "items": {
-                "type": "object"
-            }
-        },
-        "edata": {
-            "id": "http://api.ekstep.org/telemetry/edata",
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-                "eks"
-            ],
-            "properties": {
-                "eks": {
-                    "id": "http://api.ekstep.org/telemetry/edata/eks",
-                    "type": "object",
-                    "additionalProperties": true,
-                    "properties": {
-                        "uid": {
-                            "type": "string"
-                        },
-                        "handle": {
-                            "type": "string"
-                        },
-                        "gender": {
-                            "type": "string"
-                        },
-                        "age": {
-                            "type": "number"
-                        },
-                        "standard": {
-                            "type": "number"
-                        },
-                        "language": {
-                            "type": "string"
-                        },
-                        "day": {
-                            "type": "number"
-                        },
-                        "month": {
-                            "type": "number"
-                        }
-                    }
-                }
-            }
-        }
-    }
-}
\ No newline at end of file
diff --git a/schemas/2.2/ge_view_progress.json b/schemas/2.2/ge_view_progress.json
deleted file mode 100644
index deb1c29dae..0000000000
--- a/schemas/2.2/ge_view_progress.json
+++ /dev/null
@@ -1,56 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/ge_view_progress",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "GE_VIEW_PROGRESS"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "ueksid"
-                            ],
-                            "properties": {
-                                "ueksid": {
-                                    "type": "string",
-                                    "minLength": 1
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_assess.json b/schemas/2.2/oe_assess.json
deleted file mode 100644
index 33d0313327..0000000000
--- a/schemas/2.2/oe_assess.json
+++ /dev/null
@@ -1,136 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "id": "http://api.ekstep.org/telemetry/oe_assess",
-    "type": "object",
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_ASSESS"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "qid",
-                                "pass",
-                                "score",
-                                "resvalues",
-                                "length",
-                                "exlength",
-                                "params",
-                                "uri",
-                                "qindex"
-                            ],
-                            "properties": {
-                                "qid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/qid",
-                                    "type": "string"
-                                },
-                                "pass": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/pass",
-                                    "enum": [
-                                        "Yes",
-                                        "No"
-                                    ]
-                                },
-                                "score": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/score",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "resvalues": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/res",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                },
-                                "length": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/length",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "exlength": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/exlength",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "params": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/params",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                },
-                                "uri": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/uri",
-                                    "type": "string"
-                                },
-                                "qindex": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/qindex",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "qtitle": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                                    "type": "string"
-                                },
-                                "qdesc": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/qtitle",
-                                    "type": "string"
-                                },
-                                "maxscore": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/maxscore",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "mc": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "string"
-                                    }
-                                },
-                                "mmc": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/mmc",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "string"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ],
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "edata",
-        "pdata",
-        "channel"
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_earn.json b/schemas/2.2/oe_earn.json
deleted file mode 100644
index e1bb751be7..0000000000
--- a/schemas/2.2/oe_earn.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_earn",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_EARN"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "required": [
-                                "type",
-                                "points"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "MONEY",
-                                        "GEMS",
-                                        "POINTS"
-                                    ]
-                                },
-                                "points": {
-                                    "type": "number",
-                                    "minimum": 0
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_end.json b/schemas/2.2/oe_end.json
deleted file mode 100644
index c2a9cdbe8f..0000000000
--- a/schemas/2.2/oe_end.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_end",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_END"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "length"
-                            ],
-                            "properties": {
-                                "length": {
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "progress": {
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "stageid": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_error.json b/schemas/2.2/oe_error.json
deleted file mode 100644
index 7ddab3828e..0000000000
--- a/schemas/2.2/oe_error.json
+++ /dev/null
@@ -1,102 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_error",
-    "type": "object",
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_ERROR"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "env",
-                                "type",
-                                "stageid",
-                                "objecttype",
-                                "objectid",
-                                "err",
-                                "action",
-                                "data",
-                                "severity"
-                            ],
-                            "properties": {
-                                "env": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/env",
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                                    "enum": [
-                                        "SYSTEM",
-                                        "API",
-                                        "PLUGIN",
-                                        "CONTENT",
-                                        "ASSET",
-                                        "OTHER"
-                                    ]
-                                },
-                                "stageid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/stageid",
-                                    "type": "string"
-                                },
-                                "objecttype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/objecttype",
-                                    "type": "string"
-                                },
-                                "objectid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/objectid",
-                                    "type": "string"
-                                },
-                                "err": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/err",
-                                    "type": "string"
-                                },
-                                "action": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/action",
-                                    "type": "string"
-                                },
-                                "data": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/data",
-                                    "type": "string"
-                                },
-                                "severity": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/severity",
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ],
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "edata",
-        "pdata",
-        "channel"
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_feedback.json b/schemas/2.2/oe_feedback.json
deleted file mode 100644
index 5898d2dfb3..0000000000
--- a/schemas/2.2/oe_feedback.json
+++ /dev/null
@@ -1,84 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_end",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_FEEDBACK"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "rating",
-                                "context",
-                                "comments",
-                                "resvalues"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/type",
-                                    "type": "string",
-                                    "enum": [
-                                        "FLAG",
-                                        "SURVEY",
-                                        "RATING"
-                                    ]
-                                },
-                                "rating": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/rating",
-                                    "type": "number"
-                                },
-                                "context": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/context",
-                                    "type": "string"
-                                },
-                                "comments": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/comments",
-                                    "type": "string"
-                                },
-                                "resvalues": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/res",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_interact.json b/schemas/2.2/oe_interact.json
deleted file mode 100644
index 82bb414abb..0000000000
--- a/schemas/2.2/oe_interact.json
+++ /dev/null
@@ -1,121 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_interact",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_INTERACT"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "stageid",
-                                "type",
-                                "subtype",
-                                "extype",
-                                "pos",
-                                "values",
-                                "id",
-                                "tid",
-                                "uri"
-                            ],
-                            "properties": {
-                                "stageid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/type",
-                                    "type": "string",
-                                    "enum": [
-                                        "TOUCH",
-                                        "SHOW",
-                                        "HIDE",
-                                        "DRAG",
-                                        "DROP",
-                                        "PINCH",
-                                        "ZOOM",
-                                        "SHAKE",
-                                        "ROTATE",
-                                        "SPEAK",
-                                        "LISTEN",
-                                        "WRITE",
-                                        "DRAW",
-                                        "START",
-                                        "END",
-                                        "CHOOSE",
-                                        "ACTIVATE",
-                                        "OTHER",
-                                        "SCROLL",
-                                        "HEARTBEAT"
-                                    ]
-                                },
-                                "subtype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/subtype",
-                                    "type": "string"
-                                },
-                                "extype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/extype",
-                                    "type": "string"
-                                },
-                                "pos": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/pos",
-                                    "type": "array"
-                                },
-                                "values": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/values",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                },
-                                "id": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/id",
-                                    "type": "string"
-                                },
-                                "tid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/tid",
-                                    "type": "string"
-                                },
-                                "uri": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/uri",
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_interrupt.json b/schemas/2.2/oe_interrupt.json
deleted file mode 100644
index fe3c8b296b..0000000000
--- a/schemas/2.2/oe_interrupt.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_interrupt",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_INTERRUPT"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "stageid"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "type": "string",
-                                    "enum": [
-                                        "IDLE",
-                                        "SLEEP",
-                                        "CALL",
-                                        "SWITCH",
-                                        "LOCK",
-                                        "OTHER",
-                                        "BACKGROUND",
-                                        "RESUME"
-                                    ]
-                                },
-                                "stageid": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_item_response.json b/schemas/2.2/oe_item_response.json
deleted file mode 100644
index 3efd05b799..0000000000
--- a/schemas/2.2/oe_item_response.json
+++ /dev/null
@@ -1,87 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_end",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_ITEM_RESPONSE"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "qid",
-                                "type",
-                                "state",
-                                "resvalues"
-                            ],
-                            "properties": {
-                                "qid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/qid",
-                                    "type": "string"
-                                },
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/type",
-                                    "type": "string",
-                                    "enum": [
-                                        "CHOOSE",
-                                        "DRAG",
-                                        "SELECT",
-                                        "MATCH",
-                                        "INPUT",
-                                        "SPEAK",
-                                        "WRITE"
-                                    ]
-                                },
-                                "state": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/state",
-                                    "type": "string",
-                                    "enum": [
-                                        "SELECTED",
-                                        "UNSELECTED"
-                                    ]
-                                },
-                                "resvalues": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/res",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_learn.json b/schemas/2.2/oe_learn.json
deleted file mode 100644
index 10922475f8..0000000000
--- a/schemas/2.2/oe_learn.json
+++ /dev/null
@@ -1,86 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_learn",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_LEARN"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "topics"
-                            ],
-                            "properties": {
-                                "topics": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/topics",
-                                    "type": "array",
-                                    "minItems": 1,
-                                    "items": {
-                                        "type": "object",
-                                        "additionalProperties": false,
-                                        "required": [
-                                            "mc",
-                                            "methods"
-                                        ],
-                                        "properties": {
-                                            "mc": {
-                                                "id": "http://api.ekstep.org/telemetry/edata/eks/mc",
-                                                "type": "string"
-                                            },
-                                            "methods": {
-                                                "id": "http://api.ekstep.org/telemetry/edata/eks/methods",
-                                                "type": "array",
-                                                "minItems": 1,
-                                                "items": {
-                                                    "type": "string",
-                                                    "enum": [
-                                                        "PLAY",
-                                                        "ANSWER",
-                                                        "WRITE",
-                                                        "SPEAK",
-                                                        "OTHER"
-                                                    ]
-                                                }
-                                            }
-                                        }
-                                    }
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_level_set.json b/schemas/2.2/oe_level_set.json
deleted file mode 100644
index 3a9dc1c3da..0000000000
--- a/schemas/2.2/oe_level_set.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_level_set",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_LEVEL_SET"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "category",
-                                "current",
-                                "max"
-                            ],
-                            "properties": {
-                                "category": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/category",
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "current": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/current",
-                                    "type": "string",
-                                    "minLength": 1
-                                },
-                                "max": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/max",
-                                    "type": "string",
-                                    "minLength": 1
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_navigate.json b/schemas/2.2/oe_navigate.json
deleted file mode 100644
index 0330d70194..0000000000
--- a/schemas/2.2/oe_navigate.json
+++ /dev/null
@@ -1,70 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_end",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_NAVIGATE"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "type",
-                                "itype",
-                                "stageid",
-                                "stageto"
-                            ],
-                            "properties": {
-                                "type": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/type",
-                                    "type": "string"
-                                },
-                                "itype": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/itype",
-                                    "type": "string"
-                                },
-                                "stageid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/stageid",
-                                    "type": "string"
-                                },
-                                "stageto": {
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_start.json b/schemas/2.2/oe_start.json
deleted file mode 100644
index 599487c742..0000000000
--- a/schemas/2.2/oe_start.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_start",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_START"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "properties": {
-                                "mode": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/mode",
-                                    "type": "string"
-                                },
-                                "stageid": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/stageid",
-                                    "type": "string"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_summary.json b/schemas/2.2/oe_summary.json
deleted file mode 100644
index 08c86a17bc..0000000000
--- a/schemas/2.2/oe_summary.json
+++ /dev/null
@@ -1,210 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_earn",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata",
-        "pdata",
-        "channel"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_SUMMARY"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "required": [
-                                "timespent"
-                            ],
-                            "properties": {
-                                "timespent": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/timespent",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "interrupts": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/interrupts",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "screensummary": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    },
-                                    "properties": {
-                                        "id": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/id",
-                                            "type": "string"
-                                        },
-                                        "timespent": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/timespent",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "count": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/screensummary/count",
-                                            "type": "number",
-                                            "minimum": 0
-                                        }
-                                    }
-                                },
-                                "eventsummary": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    },
-                                    "properties": {
-                                        "id": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/id",
-                                            "type": "string"
-                                        },
-                                        "count": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/eventsummary/count",
-                                            "type": "number",
-                                            "minimum": 0
-                                        }
-                                    }
-                                },
-                                "items": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/items",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    },
-                                    "properties": {
-                                        "qid": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/qid",
-                                            "type": "string"
-                                        },
-                                        "pass": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/pass",
-                                            "type": "string"
-                                        },
-                                        "maxscore": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/maxscore",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "score": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/score",
-                                            "type": "string"
-                                        },
-                                        "resvalues": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/resvalues",
-                                            "type": "array",
-                                            "items": {
-                                                "type": "object"
-                                            }
-                                        },
-                                        "length": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/length",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "qindex": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/qindex",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "qdesc": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/qdesc",
-                                            "type": "string"
-                                        },
-                                        "qtitle": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/qtitle",
-                                            "type": "string"
-                                        },
-                                        "mmc": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/mmc",
-                                            "type": "array",
-                                            "items": {
-                                                "type": "string"
-                                            }
-                                        },
-                                        "mc": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/items/mc",
-                                            "type": "array",
-                                            "items": {
-                                                "type": "string"
-                                            }
-                                        }
-                                    }
-                                },
-                                "itemsummary": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary",
-                                    "type": "array",
-                                    "items": {
-                                        "type": "object"
-                                    },
-                                    "properties": {
-                                        "qid": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/qid",
-                                            "type": "string"
-                                        },
-                                        "score": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/score",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "attempts": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/attempts",
-                                            "type": "number",
-                                            "minimum": 0
-                                        },
-                                        "timespent": {
-                                            "id": "http://api.ekstep.org/telemetry/edata/eks/itemsummary/timespent",
-                                            "type": "number",
-                                            "minimum": 0
-                                        }
-                                    }
-                                },
-                                "interactevents": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/interactevents",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "interacteventspermin": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/interacteventspermin",
-                                    "type": "number",
-                                    "minimum": 0
-                                },
-                                "progress": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/progress",
-                                    "type": "number",
-                                    "minimum": 0
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/oe_xapi.json b/schemas/2.2/oe_xapi.json
deleted file mode 100644
index 04c1e9e546..0000000000
--- a/schemas/2.2/oe_xapi.json
+++ /dev/null
@@ -1,54 +0,0 @@
-{
-    "id": "http://api.ekstep.org/telemetry/oe_xapi",
-    "type": "object",
-    "required": [
-        "eid",
-        "ets",
-        "ver",
-        "gdata",
-        "sid",
-        "uid",
-        "did",
-        "mid",
-        "edata"
-    ],
-    "allOf": [
-        {
-            "$ref": "http://localhost:7070/schemas/2.2/common.json"
-        },
-        {
-            "properties": {
-                "eid": {
-                    "id": "http://api.ekstep.org/telemetry/eid",
-                    "enum": [
-                        "OE_XAPI"
-                    ]
-                },
-                "edata": {
-                    "id": "http://api.ekstep.org/telemetry/edata",
-                    "type": "object",
-                    "additionalProperties": false,
-                    "required": [
-                        "eks"
-                    ],
-                    "properties": {
-                        "eks": {
-                            "id": "http://api.ekstep.org/telemetry/edata/eks",
-                            "type": "object",
-                            "additionalProperties": false,
-                            "required": [
-                                "xapi"
-                            ],
-                            "properties": {
-                                "xapi": {
-                                    "id": "http://api.ekstep.org/telemetry/edata/eks/xapi",
-                                    "type": "object"
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-        }
-    ]
-}
\ No newline at end of file
diff --git a/schemas/2.2/pdata.json b/schemas/2.2/pdata.json
deleted file mode 100644
index cd509bc58b..0000000000
--- a/schemas/2.2/pdata.json
+++ /dev/null
@@ -1,15 +0,0 @@
-{
-      "type": "object",
-      "properties": {
-        "id": {
-          "id": "http://api.ekstep.org/telemetry/pdata/id",
-          "type": "string"
-        },
-        "ver": {
-          "id": "http://api.ekstep.org/telemetry/pdata/ver",
-          "type": "string"
-        }
-      },
-      "additionalProperties": false,
-      "required": ["id","ver"]
-}
diff --git a/schemas/3.0/actor.json b/schemas/3.0/actor.json
deleted file mode 100644
index cd87b3fb22..0000000000
--- a/schemas/3.0/actor.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/actor/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/actor/type",
-      "type": "string"
-    }
-  },
-  "required": [ "type", "id"]
-}
\ No newline at end of file
diff --git a/schemas/3.0/assess.json b/schemas/3.0/assess.json
deleted file mode 100644
index dd5e066ccb..0000000000
--- a/schemas/3.0/assess.json
+++ /dev/null
@@ -1,68 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/assess",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "ASSESS"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "item",
-            "pass",
-            "score",
-            "resvalues",
-            "duration"
-          ],
-          "properties": {
-            "item": {
-              "$ref": "http://localhost:7070/schemas/3.0/question.json"
-            },
-            "index": {
-              "id": "http://api.ekstep.org/telemetry/edata/index",
-              "type": "number"
-            },
-            "pass": {
-              "id": "http://api.ekstep.org/telemetry/edata/pass",
-              "type": "string"
-            },
-            "score": {
-              "id": "http://api.ekstep.org/telemetry/edata/score",
-              "type": "number"
-            },
-            "resvalues": {
-              "id": "http://api.ekstep.org/telemetry/edata/resvalues",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/audit.json b/schemas/3.0/audit.json
deleted file mode 100644
index bf8fea1b04..0000000000
--- a/schemas/3.0/audit.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/audit",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "AUDIT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "props": {
-              "id": "http://api.ekstep.org/telemetry/edata/props",
-              "type": "array",
-              "items": {
-                "type": "string"
-              }
-            },
-            "state": {
-              "id": "http://api.ekstep.org/telemetry/edata/state",
-              "type": "string"
-            },
-            "prevstate": {
-              "id": "http://api.ekstep.org/telemetry/edata/prevstate",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/cdata.json b/schemas/3.0/cdata.json
deleted file mode 100644
index fd82b2b534..0000000000
--- a/schemas/3.0/cdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-      "type": "array",
-      "items": {
-        "type": "object",
-        "properties": {
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/cdata/type",
-            "type": "string"
-          },
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/cdata/id",
-            "type": "string"
-          }
-        },
-        "additionalProperties": false,
-        "required": [ "type", "id"]
-      }
-}
\ No newline at end of file
diff --git a/schemas/3.0/common.json b/schemas/3.0/common.json
deleted file mode 100644
index 4228704830..0000000000
--- a/schemas/3.0/common.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "ets": {
-      "id": "http://api.ekstep.org/telemetry/ets",
-      "type": "number",
-      "format": "date-time"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/ver",
-      "type": "string",
-      "enum": [
-        "3.0",
-        "3.1"
-      ]
-    },
-    "mid": {
-      "id": "http://api.ekstep.org/telemetry/mid",
-      "type": "string",
-      "minLength": 1
-    },
-    "actor": {
-      "$ref": "http://localhost:7070/schemas/3.0/actor.json"
-    },
-    "context": {
-      "$ref": "http://localhost:7070/schemas/3.0/context.json"
-    },
-    "object": {
-      "$ref": "http://localhost:7070/schemas/3.0/object.json"
-    },
-    "tags": {
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/context.json b/schemas/3.0/context.json
deleted file mode 100644
index 95f3f2cc7c..0000000000
--- a/schemas/3.0/context.json
+++ /dev/null
@@ -1,33 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "channel": {
-      "id": "http://api.ekstep.org/telemetry/context/channel",
-      "type": "string",
-      "minLength": 1
-    },
-    "pdata": {
-      "$ref": "http://localhost:7070/schemas/3.0/pdata.json"
-    },
-    "env": {
-      "id": "http://api.ekstep.org/telemetry/context/env",
-      "type": "string"
-    },
-    "sid": {
-      "id": "http://api.ekstep.org/telemetry/context/sid",
-      "type": "string"
-    },
-    "did": {
-      "id": "http://api.ekstep.org/telemetry/context/did",
-      "type": "string"
-    },
-    "cdata": {
-      "$ref": "http://localhost:7070/schemas/3.0/cdata.json"
-    }
-  },
-  "required": [
-      "channel",
-      "env",
-      "pdata"
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/dspec.json b/schemas/3.0/dspec.json
deleted file mode 100644
index cc9ee16701..0000000000
--- a/schemas/3.0/dspec.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "os": {
-      "id": "http://api.ekstep.org/telemetry/dspec/os",
-      "type": "string"
-    },
-    "make": {
-      "id": "http://api.ekstep.org/telemetry/dspec/make",
-      "type": "string"
-    },
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/dspec/id",
-      "type": "string"
-    },
-    "mem": {
-      "id": "http://api.ekstep.org/telemetry/dspec/mem",
-      "type": "number",
-      "minimum": -1
-    },
-    "idisk": {
-      "id": "http://api.ekstep.org/telemetry/dspec/idisk",
-      "type": "number",
-      "minimum": -1
-    },
-    "edisk": {
-      "id": "http://api.ekstep.org/telemetry/dspec/edisk",
-      "type": "number",
-      "minimum": -1
-    },
-    "scrn": {
-      "id": "http://api.ekstep.org/telemetry/dspec/scrn",
-      "type": "number",
-      "minimum": -1
-    },
-    "camera": {
-      "id": "http://api.ekstep.org/telemetry/dspec/camera",
-      "type": "string"
-    },
-    "cpu": {
-      "id": "http://api.ekstep.org/telemetry/dspec/cpu",
-      "type": "string"
-    },
-    "sims": {
-      "id": "http://api.ekstep.org/telemetry/dspec/sims",
-      "type": "number",
-      "minimum": -1
-    },
-    "cap": {
-      "id": "http://api.ekstep.org/telemetry/dspec/cap",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/end.json b/schemas/3.0/end.json
deleted file mode 100644
index 2805c2f288..0000000000
--- a/schemas/3.0/end.json
+++ /dev/null
@@ -1,61 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/end",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "END"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "mode": {
-              "id": "http://api.ekstep.org/telemetry/edata/mode",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "summary": {
-              "id": "http://api.ekstep.org/telemetry/edata/summary",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/envelope.json b/schemas/3.0/envelope.json
deleted file mode 100644
index 76ecbf8fde..0000000000
--- a/schemas/3.0/envelope.json
+++ /dev/null
@@ -1,27 +0,0 @@
-{
-  "id":"http://api.ekstep.org/telemetry/envelope",
-  "type":"object",
-  "required":[
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf":[
-    {
-      "$ref":"http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties":{
-        "eid":{
-          "id":"http://api.ekstep.org/telemetry/mid",
-          "type":"string",
-          "minLength": 1
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/error.json b/schemas/3.0/error.json
deleted file mode 100644
index 27cff116b7..0000000000
--- a/schemas/3.0/error.json
+++ /dev/null
@@ -1,62 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/error",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "ERROR"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "err",
-            "errtype",
-            "stacktrace"
-          ],
-          "properties": {
-            "err": {
-              "id": "http://api.ekstep.org/telemetry/edata/err",
-              "type": "string"
-            },
-            "errtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/errtype",
-              "type": "string"
-            },
-            "stacktrace": {
-              "id": "http://api.ekstep.org/telemetry/edata/stacktrace",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "object": {
-              "$ref": "http://localhost:7070/schemas/3.0/inlineobject.json"
-            },
-            "plugin": {
-              "$ref": "http://localhost:7070/schemas/3.0/plugin.json"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/event.json b/schemas/3.0/event.json
deleted file mode 100644
index 656882247a..0000000000
--- a/schemas/3.0/event.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
-  "$schema": "http://json-schema.org/draft-04/schema#",
-  "id": "http://api.ekstep.org/telemetry/event",
-  "type": "object",
-  "oneOf": [{
-      "$ref": "http://localhost:7070/schemas/3.0/start.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/end.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/impression.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/interact.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/assess.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/response.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/interrupt.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/feedback.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/share.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/audit.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/error.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/heartbeat.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/log.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/search.json"
-    },
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/exdata.json"
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/exdata.json b/schemas/3.0/exdata.json
deleted file mode 100644
index 2d3ed99b40..0000000000
--- a/schemas/3.0/exdata.json
+++ /dev/null
@@ -1,43 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/exdata",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "EXDATA"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "data": {
-              "id": "http://api.ekstep.org/telemetry/edata/data",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/feedback.json b/schemas/3.0/feedback.json
deleted file mode 100644
index c453c17ecf..0000000000
--- a/schemas/3.0/feedback.json
+++ /dev/null
@@ -1,51 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/feedback",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "FEEDBACK"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "properties": {
-            "rating": {
-              "id": "http://api.ekstep.org/telemetry/edata/rating",
-              "type": "number"
-            },
-            "comments": {
-              "id": "http://api.ekstep.org/telemetry/edata/comments",
-              "type": "string"
-            },
-            "commentid": {
-              "id": "http://api.ekstep.org/telemetry/edata/commentid",
-              "type": "string"
-            },
-            "commenttxt": {
-              "id": "http://api.ekstep.org/telemetry/edata/commenttxt",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/heartbeat.json b/schemas/3.0/heartbeat.json
deleted file mode 100644
index 20fa50cd73..0000000000
--- a/schemas/3.0/heartbeat.json
+++ /dev/null
@@ -1,32 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/heartbeat",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "HEARTBEAT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object"
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/impression.json b/schemas/3.0/impression.json
deleted file mode 100644
index 7f1b8623be..0000000000
--- a/schemas/3.0/impression.json
+++ /dev/null
@@ -1,94 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/impression",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "IMPRESSION"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "pageid",
-            "uri"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "subtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/subtype",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-             },
-            "uri": {
-              "id": "http://api.ekstep.org/telemetry/edata/uri",
-              "type": "string"
-            },
-            "visits": {
-              "id": "http://api.ekstep.org/telemetry/edata/visits",
-              "type": "array",
-              "items": {
-                "type": "object",
-                "properties": {
-                  "objid": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objid",
-                    "type": "string"
-                  },
-                  "objtype": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objtype",
-                    "type": "string"
-                  },
-                  "objver": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/objver",
-                    "type": "string"
-                  },
-                  "section": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/section",
-                    "type": "string"
-                  },
-                  "index": {
-                    "id": "http://api.ekstep.org/telemetry/edata/visits/index",
-                    "type": "number"
-                  }
-                },
-                "additionalProperties": false,
-                "required": [
-                  "objid",
-                  "objtype"
-                ]
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/inlineobject.json b/schemas/3.0/inlineobject.json
deleted file mode 100644
index 69c5e6abd3..0000000000
--- a/schemas/3.0/inlineobject.json
+++ /dev/null
@@ -1,38 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "type",
-    "ver"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/type",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/ver",
-      "type": "string"
-    },
-    "subtype": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/subtype",
-      "type": "string"
-    },
-    "name": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/name",
-      "type": "string"
-    },
-    "code": {
-      "id": "http://api.ekstep.org/telemetry/inlineobject/code",
-      "type": "string"
-    },
-    "parent": {
-      "$ref": "http://localhost:7070/schemas/3.0/parent.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/interact.json b/schemas/3.0/interact.json
deleted file mode 100644
index 13e600d9d9..0000000000
--- a/schemas/3.0/interact.json
+++ /dev/null
@@ -1,69 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/interact",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "INTERACT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "id"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "subtype": {
-              "id": "http://api.ekstep.org/telemetry/edata/subtype",
-              "type": "string"
-            },
-            "id": {
-              "id": "http://api.ekstep.org/telemetry/edata/id",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "target": {
-              "$ref": "http://localhost:7070/schemas/3.0/target.json"
-            },
-            "plugin": {
-              "$ref": "http://localhost:7070/schemas/3.0/plugin.json"
-            },
-            "extra": {
-              "id": "http://api.ekstep.org/telemetry/edata/extra",
-              "type": "object"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/interrupt.json b/schemas/3.0/interrupt.json
deleted file mode 100644
index 6f8835ba82..0000000000
--- a/schemas/3.0/interrupt.json
+++ /dev/null
@@ -1,46 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/interrupt",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "INTERRUPT"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/items.json b/schemas/3.0/items.json
deleted file mode 100644
index 937cd76555..0000000000
--- a/schemas/3.0/items.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/items",
-  "type": "array",
-  "items": {
-    "type": "object",
-    "additionalProperties": false,
-    "properties": {
-      "id": {
-        "id": "http://api.ekstep.org/telemetry/items/id",
-        "type": "string"
-      },
-      "type": {
-        "id": "http://api.ekstep.org/telemetry/items/type",
-        "type": "string"
-      },
-      "ver": {
-        "id": "http://api.ekstep.org/telemetry/items/ver",
-        "type": "string"
-      },
-      "params": {
-        "id": "http://api.ekstep.org/telemetry/items/params",
-        "type": "array",
-        "items": {
-          "type": "object"
-        }
-      },
-      "origin": {
-        "id": "http://api.ekstep.org/telemetry/items/origin",
-        "type": "object",
-        "properties": {
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/items/origin/id",
-            "type": "string"
-          },
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/items/origin/type",
-            "type": "string"
-          }
-        }
-      },
-      "to": {
-        "id": "http://api.ekstep.org/telemetry/items/to",
-        "type": "object",
-        "properties": {
-          "id": {
-            "id": "http://api.ekstep.org/telemetry/items/to/id",
-            "type": "string"
-          },
-          "type": {
-            "id": "http://api.ekstep.org/telemetry/items/to/type",
-            "type": "string"
-          }
-        }
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/log.json b/schemas/3.0/log.json
deleted file mode 100644
index e29b1a03bc..0000000000
--- a/schemas/3.0/log.json
+++ /dev/null
@@ -1,63 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/log",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "LOG"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type",
-            "level",
-            "message"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "level": {
-              "id": "http://api.ekstep.org/telemetry/edata/level",
-              "type": "string"
-            },
-            "message": {
-              "id": "http://api.ekstep.org/telemetry/edata/message",
-              "type": "string"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            },
-            "params": {
-              "id": "http://api.ekstep.org/telemetry/edata/params",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/object.json b/schemas/3.0/object.json
deleted file mode 100644
index 4adffacb39..0000000000
--- a/schemas/3.0/object.json
+++ /dev/null
@@ -1,20 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/object/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/object/type",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/object/ver",
-      "type": "string"
-    },
-    "rollup": {
-      "$ref": "http://localhost:7070/schemas/3.0/rollup.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/parent.json b/schemas/3.0/parent.json
deleted file mode 100644
index 07d4262af7..0000000000
--- a/schemas/3.0/parent.json
+++ /dev/null
@@ -1,13 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/parent/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/parent/type",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/pdata.json b/schemas/3.0/pdata.json
deleted file mode 100644
index 18f9ba13b6..0000000000
--- a/schemas/3.0/pdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/pdata/id",
-      "type": "string"
-    },
-    "pid": {
-      "id": "http://api.ekstep.org/telemetry/pdata/pid",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/pdata/ver",
-      "type": "string"
-    }
-  },
-  "required": ["id"]
-}
\ No newline at end of file
diff --git a/schemas/3.0/plugin.json b/schemas/3.0/plugin.json
deleted file mode 100644
index cfd7d80ba9..0000000000
--- a/schemas/3.0/plugin.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "ver"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/plugin/id",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/plugin/ver",
-      "type": "string"
-    },
-    "category": {
-      "id": "http://api.ekstep.org/telemetry/plugin/category",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/question.json b/schemas/3.0/question.json
deleted file mode 100644
index 89bd24f052..0000000000
--- a/schemas/3.0/question.json
+++ /dev/null
@@ -1,58 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/question/id",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/question/type",
-      "type": "string"
-    },
-    "maxscore": {
-      "id": "http://api.ekstep.org/telemetry/question/maxscore",
-      "type": "number"
-    },
-    "exlength": {
-      "id": "http://api.ekstep.org/telemetry/question/exlength",
-      "type": "number"
-    },
-    "params": {
-      "id": "http://api.ekstep.org/telemetry/question/params",
-      "type": "array",
-      "items": {
-        "type": "object"
-      }
-    },
-    "uri": {
-      "id": "http://api.ekstep.org/telemetry/question/uri",
-      "type": "string"
-    },
-    "desc": {
-      "id": "http://api.ekstep.org/telemetry/question/desc",
-      "type": "string"
-    },
-    "title": {
-      "id": "http://api.ekstep.org/telemetry/question/title",
-      "type": "string"
-    },
-    "mmc": {
-      "id": "http://api.ekstep.org/telemetry/question/mmc",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    },
-    "mc": {
-      "id": "http://api.ekstep.org/telemetry/question/mc",
-      "type": "array",
-      "items": {
-        "type": "string"
-      }
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/response.json b/schemas/3.0/response.json
deleted file mode 100644
index d2b3630e7f..0000000000
--- a/schemas/3.0/response.json
+++ /dev/null
@@ -1,53 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "RESPONSE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "target",
-            "type",
-            "values"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/type",
-              "type": "string"
-            },
-            "target": {
-              "$ref": "http://localhost:7070/schemas/3.0/target.json"
-            },
-            "values": {
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/rollup.json b/schemas/3.0/rollup.json
deleted file mode 100644
index bfd89209f2..0000000000
--- a/schemas/3.0/rollup.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "l1": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l1",
-      "type": "string"
-    },
-    "l2": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l2",
-      "type": "string"
-    },
-    "l3": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l3",
-      "type": "string"
-    },
-    "l4": {
-      "id": "http://api.ekstep.org/telemetry/context/rollup/l4",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/search.json b/schemas/3.0/search.json
deleted file mode 100644
index e5858bccf4..0000000000
--- a/schemas/3.0/search.json
+++ /dev/null
@@ -1,71 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/search",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "SEARCH"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "query",
-            "size",
-            "topn"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "query": {
-              "id": "http://api.ekstep.org/telemetry/edata/query",
-              "type": "string"
-            },
-            "filters": {
-              "id": "http://api.ekstep.org/telemetry/edata/filters",
-              "type": "object"
-            },
-            "sort": {
-              "id": "http://api.ekstep.org/telemetry/edata/sort",
-              "type": "object"
-            },
-            "correlationid": {
-              "id": "http://api.ekstep.org/telemetry/edata/correlationid",
-              "type": "string"
-            },
-            "size": {
-              "id": "http://api.ekstep.org/telemetry/edata/size",
-              "type": "number"
-            },
-            "topn": {
-              "id": "http://api.ekstep.org/telemetry/edata/topn",
-              "type": "array",
-              "items": {
-                "type": "object"
-              }
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/share.json b/schemas/3.0/share.json
deleted file mode 100644
index 880df92ee5..0000000000
--- a/schemas/3.0/share.json
+++ /dev/null
@@ -1,49 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/share",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "SHARE"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "items"
-          ],
-          "properties": {
-            "dir": {
-              "id": "http://api.ekstep.org/telemetry/edata/dir",
-              "type": "string"
-            },
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "items": {
-              "$ref": "http://localhost:7070/schemas/3.0/items.json"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/start.json b/schemas/3.0/start.json
deleted file mode 100644
index d274b124a7..0000000000
--- a/schemas/3.0/start.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
-  "id": "http://api.ekstep.org/telemetry/start",
-  "type": "object",
-  "required": [
-    "eid",
-    "ets",
-    "ver",
-    "mid",
-    "actor",
-    "context",
-    "edata"
-  ],
-  "allOf": [
-    {
-      "$ref": "http://localhost:7070/schemas/3.0/common.json"
-    },
-    {
-      "properties": {
-        "eid": {
-          "id": "http://api.ekstep.org/telemetry/eid",
-          "enum": [
-            "START"
-          ]
-        },
-        "edata": {
-          "id": "http://api.ekstep.org/telemetry/edata",
-          "type": "object",
-          "additionalProperties": false,
-          "required": [
-            "type"
-          ],
-          "properties": {
-            "type": {
-              "id": "http://api.ekstep.org/telemetry/edata/type",
-              "type": "string"
-            },
-            "dspec": {
-              "$ref": "http://localhost:7070/schemas/3.0/dspec.json"
-            },
-            "uaspec": {
-              "$ref": "http://localhost:7070/schemas/3.0/uaspec.json"
-            },
-            "loc": {
-              "id": "http://api.ekstep.org/telemetry/edata/loc",
-              "type": "string"
-            },
-            "mode": {
-              "id": "http://api.ekstep.org/telemetry/edata/mode",
-              "type": "string"
-            },
-            "duration": {
-              "id": "http://api.ekstep.org/telemetry/edata/duration",
-              "type": "number"
-            },
-            "pageid": {
-              "id": "http://api.ekstep.org/telemetry/edata/pageid",
-              "type": "string"
-            }
-          }
-        }
-      }
-    }
-  ]
-}
\ No newline at end of file
diff --git a/schemas/3.0/target.json b/schemas/3.0/target.json
deleted file mode 100644
index 363c7103b0..0000000000
--- a/schemas/3.0/target.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "id",
-    "ver",
-    "type"
-  ],
-  "properties": {
-    "id": {
-      "id": "http://api.ekstep.org/telemetry/target/id",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/target/ver",
-      "type": "string"
-    },
-    "type": {
-      "id": "http://api.ekstep.org/telemetry/target/type",
-      "type": "string"
-    },
-    "parent": {
-      "$ref": "http://localhost:7070/schemas/3.0/parent.json"
-    }
-  }
-}
\ No newline at end of file
diff --git a/schemas/3.0/uaspec.json b/schemas/3.0/uaspec.json
deleted file mode 100644
index cc7d9f53c6..0000000000
--- a/schemas/3.0/uaspec.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
-  "type": "object",
-  "properties": {
-    "agent": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/agent",
-      "type": "string"
-    },
-    "ver": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/ver",
-      "type": "string"
-    },
-    "system": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/system",
-      "type": "string"
-    },
-    "platform": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/platform",
-      "type": "string"
-    },
-    "raw": {
-      "id": "http://api.ekstep.org/telemetry/uaspec/raw",
-      "type": "string"
-    }
-  }
-}
\ No newline at end of file