Patch 1

.circleci/config.yml

@@ -2,7 +2,8 @@ version: 2.1
 jobs:
   datapipeline-build:
     working_directory: ~/dp/data-pipeline-flink
-    machine: true
+    machine:
+      image: ubuntu-2004:202008-01
     steps:
       - checkout:
           path: ~/dp
@@ -22,4 +23,4 @@ workflows:
   build_and_test:
     jobs:
       - datapipeline-build
-#testing quality gate
+#testing  quality gate

.github/pull_request_template.md

@@ -0,0 +1,32 @@
+Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
+
+### Type of change
+
+Please choose appropriate options.
+
+- [ ] Bug fix (non-breaking change which fixes an issue)
+- [ ] New feature (non-breaking change which adds functionality)
+- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] This change requires a documentation update
+
+### How Has This Been Tested?
+
+Please describe the tests that you ran to verify your changes in the below checkboxes. Provide instructions so we can reproduce. Please also list any relevant details for your test configuration
+
+- [ ] Ran Test A
+- [ ] Ran Test B
+
+**Test Configuration**:
+* Software versions:
+* Hardware versions:
+
+### Checklist:
+
+- [ ] My code follows the style guidelines of this project
+- [ ] I have performed a self-review of my own code
+- [ ] I have commented my code, particularly in hard-to-understand areas
+- [ ] I have made corresponding changes to the documentation
+- [ ] My changes generate no new warnings
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] New and existing unit tests pass locally with my changes
+- [ ] Any dependent changes have been merged and published in downstream modules

.github/workflows/jira-description-action.yml

@@ -0,0 +1,16 @@
+name: jira-description-action
+on:
+  pull_request_target:
+    types: [opened, labeled]
+jobs:
+  add-jira-description:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: project-sunbird/[email protected]
+        name: jira-description-action
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          jira-token: ${{ secrets.JIRA_TOKEN }}
+          jira-base-url: ${{ secrets.JIRA_BASE_URL }}
+          fail-when-jira-issue-not-found: ${{ secrets.FAIL_WHEN_JIRA_ISSUE_NOT_FOUND }}
+          use: both

.gitignore

@@ -12,6 +12,7 @@ database.yml
 .idea
 tmp/
 kibana-auth/node_modules/
+kubernetes/helm_charts/*/values.yaml
 
 # DevOps
 *.backup

ansible/artifacts-download.yml

@@ -3,8 +3,40 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-download-azure
+  tasks:
+    - name: download artifact from azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-download.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: download artifact from gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: download.yml
+      vars:
+        gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+        gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: download artifact from aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: download.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"

ansible/artifacts-upload.yml

@@ -3,8 +3,41 @@
   become: yes
   vars_files:
     - "{{inventory_dir}}/secrets.yml"
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_artifact_storage_account_name}}"
-    AZURE_STORAGE_SAS_TOKEN: "{{sunbird_artifact_storage_account_sas}}"
-  roles:
-    - artifacts-upload-azure
+  tasks:
+    - name: upload artifact to azure storage
+      include_role:
+        name: azure-cloud-storage
+        tasks_from: blob-upload.yml
+      vars:
+        blob_container_name: "{{ cloud_storage_artifacts_bucketname }}"
+        container_public_access: "off"
+        blob_file_name: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+        storage_account_name: "{{ cloud_artifact_storage_accountname }}"
+        storage_account_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "azure"
+
+    - name: upload artifact to gcloud storage
+      include_role:
+        name: gcp-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        gcp_storage_service_account_name: "{{ cloud_artifact_storage_accountname }}"
+        gcp_storage_key_file: "{{ cloud_artifact_storage_secret }}"
+        gcp_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        gcp_path: "{{ artifact }}"
+        local_file_or_folder_path: "{{ artifact_path }}"
+      when: cloud_service_provider == "gcloud"
+
+    - name: upload artifact to aws s3
+      include_role:
+        name: aws-cloud-storage
+        tasks_from: upload.yml
+      vars:
+        local_file_or_folder_path: "{{ artifact_path }}"
+        s3_bucket_name: "{{ cloud_storage_artifacts_bucketname }}"
+        s3_path: "{{ artifact }}"
+        aws_default_region: "{{ cloud_public_storage_region }}"
+        aws_access_key_id: "{{ cloud_artifact_storage_accountname }}"
+        aws_secret_access_key: "{{ cloud_artifact_storage_secret }}"
+      when: cloud_service_provider == "aws"

ansible/azure-hdinsight-spark.provision.yml

@@ -0,0 +1,18 @@
+- hosts: local
+  become: yes
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - azure-hdinsight-spark-cluster
+  tags:
+    - copy-script
+
+- hosts: spark-hdinsight-cluster
+  become: yes
+  gather_facts: no
+  vars_files:
+    - "{{inventory_dir}}/secrets.yml"
+  roles:
+    - provision-azure-spark-cluster
+  tags:
+    - spark-provision

ansible/cassandra-backup.yml

@@ -2,8 +2,5 @@
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
   roles:
    - cassandra-backup

ansible/cassandra-restore.yml

@@ -2,9 +2,5 @@
   become: yes
   vars_files:
     - ['{{inventory_dir}}/secrets.yml']
-  environment:
-    AZURE_STORAGE_ACCOUNT: "{{sunbird_management_storage_account_name}}"
-    AZURE_STORAGE_KEY: "{{sunbird_management_storage_account_key}}"
-
   roles:
    - cassandra-restore

ansible/dp_kafka_setup.yml

@@ -4,7 +4,7 @@
   pre_tasks:
     - name: Registering kafka_id
       set_fact:
-        kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+        kafka_id: "{% for servername in ansible_play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
     - name: Print kafka_id
       debug:
         var: kafka_id

ansible/druid-ingestion.yml

@@ -1,6 +1,36 @@
 ---
+- hosts: localhost
+  tasks:
+   - name: set_fact
+     set_fact:
+       single_cluster: true
+     tags: populate_var
 
-- hosts: overlord
+   - name: check for hosts
+     set_fact:
+       single_cluster: false
+     when: "'rollup-overlord' in groups"
+     tags: populate_var
+
+- hosts: rollup-overlord
+  become: yes
+  roles:
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ rollup_ingestion_task_names }}"
+        overlord_host: "{{ rollup_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+
+- hosts: raw-overlord
   become: yes
   roles:
-     - druid-ingestion
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: not hostvars['localhost']['single_cluster']
+    - role: druid-ingestion
+      vars:
+        ingestion_task_names: "{{ raw_ingestion_task_names }},{{ rollup_ingestion_task_names }}"
+        overlord_host: "{{ raw_overlord_host }}"
+      when: hostvars['localhost']['single_cluster']

ansible/druid-provision.yml

@@ -1,38 +1,18 @@
 -
   name: Install Java
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
   roles: 
-    - jdk-1.8.0_121
+    - {role: jdk-1.8.0_121, when: "'java' in deploy"}
 
 -
   name: Install java and zookeeper cluster for druid
-  hosts: raw-zookeeper
+  hosts: "{{ remote }}-zookeeper"
   become: yes
   roles:
-    - {role: zookeeper-upgrade}
+    - {role: zookeeper-upgrade, when: "'zookeeper' in deploy"}
   vars:
-    zookeeper_group: "{{ groups['raw-zookeeper'] }}"
-
-#-
-#  name: Install postgres on druid cluster
-#  hosts: druid-postgres
-#  vars_files:
-#    - "{{inventory_dir}}/secrets.yml"
-#  become: yes
-#  roles:
-#    - {role: postgres-provision }
-#  vars:
-#    postgres_address_space: "0.0.0.0/0"
-#    postgresql_databases:
-#      - name: druid
-#      - name: graphite
-#    postgresql_users:
-#      - name: druid
-#        password: "{{ dp_vault_druid_postgress_pass }}"
-#      - name: graphite 
-#        password: "{{ dp_vault_druid_postgress_graphite_pass }}"
-
+    zookeeper_group: "{{ groups[remote+'-zookeeper'] }}"
 
 # Create postgres databases and users
 
@@ -43,14 +23,10 @@
   vars:
     postgresql_databases:
       - name: druid
-      - name: graphite
     postgresql_users:
       - name: druid
         db: druid
         password: "{{ dp_vault_druid_postgress_pass }}"
-      - name: graphite
-        db: graphite
-        password: "{{ dp_vault_druid_postgress_graphite_pass }}"     
   tasks:
     - name: Ensure database is created
       postgresql_db: name="{{ item.name }}" \
@@ -75,24 +51,15 @@
                   db="{{ item.db }}"
       with_items: "{{ postgresql_users }}"
 
--
-  name : Install graphite on druid
-  hosts: raw-graphite
-  become: yes
-  roles:
-    - {role: druid_graphite}
-
 -
   name: Install role druid
-  hosts: druid-raw
+  hosts: "druid-{{remote}}"
   become: yes
   roles:
-    - {role: analytics-druid, when: "'raw-coordinator' in group_names", druid_role: 'coordinator', service: 'coordinator' }
-    - {role: analytics-druid, when: "'raw-overlord' in group_names", druid_role: 'overlord', service: 'overlord'}
-    - {role: analytics-druid, when: "'raw-broker' in group_names", druid_role: 'broker', service: 'broker'}
-    - {role: analytics-druid, when: "'raw-historical' in group_names", druid_role: 'historical', service: 'historical'}
-    - {role: analytics-druid, when: "'raw-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' }
-  vars:
-    druid_postgres_db: "druid"
-    druid_zookeeper_host: "{{ groups['raw-zookeeper']|join(':2181,')}}:2181"
-    druid_graphite_host: "{{ groups['raw-graphite'][0] }}"
+    - {role: analytics-druid, when: "'{{ remote }}-coordinator' in group_names and 'coordinator' in deploy", druid_role: 'coordinator', service: 'coordinator' }
+    - {role: analytics-druid, when: "'{{ remote }}-overlord' in group_names and 'overlord' in deploy", druid_role: 'overlord', service: 'overlord'}
+    - {role: analytics-druid, when: "'{{ remote }}-broker' in group_names and 'broker' in deploy", druid_role: 'broker', service: 'broker'}
+    - {role: analytics-druid, when: "'{{ remote }}-historical' in group_names and 'historical' in deploy", druid_role: 'historical', service: 'historical'}
+    - {role: analytics-druid, when: "'{{ remote }}-middlemanager' in group_names and 'mm' in deploy", druid_role: 'middlemanager', service: 'middleManager' }
+    - {role: analytics-druid, when: "'{{ remote }}-router' in group_names and 'router' in deploy", druid_role: 'router', service: 'router' }
+

ansible/druid-rollup-provision.yml

@@ -91,5 +91,6 @@
     - {role: analytics-druid, when: "'druid-rollup-broker' in group_names", druid_role: 'broker', service: 'broker'}
     - {role: analytics-druid, when: "'druid-rollup-historical' in group_names", druid_role: 'historical', service: 'historical'}
     - {role: analytics-druid, when: "'druid-rollup-middlemanager' in group_names", druid_role: 'middlemanager', service: 'middleManager' }
+    - {role: analytics-druid, when: "'druid-rollup-router' in group_names", druid_role: 'router', service: 'router' }
   vars:
     druid_postgres_db: "druid-rollup"