Adding codebase

.gitignore

@@ -0,0 +1,12 @@
+*.backup
+*.terrahelpbkp
+.terraform
+ekstep-terra-infra/terraform.tfstate.admin
+ekstep-terra-infra/terraform.tfstate.admin.backup
+ekstep-terra-infra/terraform.tfstate.devqa
+ekstep-terra-infra/terraform.tfstate.devqa.backup
+ekstep-terra-infra/terraform.tfstate.prod
+ekstep-terra-infra/terraform.tfstate.prod.backup
+.DS_Store
+*.retry
+Session.vim

ansible.cfg

@@ -0,0 +1,3 @@
+#TODO Refactor and remove!
+[defaults]
+allow_world_readable_tmpfiles=true

es_composite_search_cluster_setup.yml

@@ -0,0 +1,50 @@
+---
+- hosts: composite-search-cluster
+  name: Setup composite search ES cluster
+  become: yes
+  pre_tasks:
+    - name: Create ecosystem group
+      become: yes
+      group:
+        state: present
+        name: "{{ ecosystem_group }}"
+        system: yes
+    - name: Create ecosystem user
+      become: yes
+      user:
+        state: present
+        name: "{{ ecosystem_user }}"
+        comment: ecosystem user
+        system: yes
+        createhome: yes
+    - name: Uncompressing and copying to system path
+      unarchive:
+        src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz
+        dest: /opt/
+        remote_src: yes
+        keep_newer: yes
+      become: yes
+      register: tar
+    - name: Updating java jre to system path
+      become: yes
+      shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999
+    - name: Registering node name
+      set_fact:
+        es_instance_name: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}cs-node-{{ loop.index }}{% endif %}{% endfor %}"
+
+  remote_user: ecosystem
+  roles:
+    - { role: es6, 
+        es_config: {
+          cluster.name: "elasticsearch-composite-search",
+          discovery.zen.ping.unicast.hosts: "{{ groups['composite-search-cluster'] }}",
+          http.port: 9200,
+          transport.tcp.port: 9300,
+          node.data: "true",
+          node.master: "true",
+          bootstrap.memory_lock: true,
+        },
+        es_heap_size: "2g",
+        es_etc_discovery_zen_ping_unicast_hosts: "{{ groups['composite-search-cluster'] }}",
+        es_etc_cluster_name: "elasticsearch-composite-search"
+      }

lp_cassandra_db_update.yml

@@ -0,0 +1,4 @@
+- hosts: lp-cassandra-new
+  become: yes
+  roles:
+    - cassandra-db-update

lp_cassandra_provision.yml

@@ -0,0 +1,14 @@
+- hosts: lp-cassandra 
+  # Install cassandra
+  become: yes
+  roles:
+    - jdk-1.8.0_121
+    - cassandra-3.7.0
+  tags: provision
+
+- hosts: lp_cassandra
+  become: yes
+  # Create Schema
+  roles:
+    - cassandra-db-update
+  tags: deploy

lp_kafka_provision.yml

@@ -0,0 +1,46 @@
+---
+- hosts: processing-cluster-kafka
+  pre_tasks:
+    - name: Create ecosytem group
+      become: yes
+      group:
+        state: present
+        name: "{{ ecosystem_group }}"
+        system: yes
+    - name: Create ecosystem user
+      become: yes
+      user:
+        state: present
+        name: "{{ ecosystem_user }}"
+        comment: ecosystem user
+        system: yes
+        createhome: yes
+        group: "{{ ecosystem_group }}"
+    - service: name=monit state=stopped
+      become: yes
+      ignore_errors: true
+    - name: Uncompressing and copying to system path
+      unarchive:
+        src: https://sunbirdpublic.blob.core.windows.net/installation/jre-8u144-linux-x64.tar.gz
+        dest: /opt/
+        remote_src: yes
+        keep_newer: yes
+      become: yes
+      register: tar
+    - name: Updating java jre to system path
+      become: yes
+      shell: update-alternatives --install /usr/bin/java java /opt/jre1.8.0_144/bin/java 9999
+    - name: Registering kafka_id
+      set_fact:
+        kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+    - name: Print kafka_id
+      debug:
+        var: kafka_id
+  become: yes
+  roles:
+    - ansible-kafka-upgrade
+  become: yes
+  post_tasks:
+   - service: name=monit state=started
+     become: yes
+     ignore_errors: true

lp_kafka_setup.yml

@@ -0,0 +1,13 @@
+---
+- hosts: processing-cluster-kafka
+  remote_user: "{{ ecosystem_user }}"
+  become: yes
+  pre_tasks:
+    - name: Registering kafka_id
+      set_fact:
+        kafka_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+    - name: Print kafka_id
+      debug:
+        var: kafka_id
+  roles:
+    - setup-kafka

lp_learning_deploy.yml

@@ -0,0 +1,10 @@
+- hosts: learningall
+  vars_files:
+    - "{{ secrets_path }}"
+  pre_tasks:
+    - name: Registering mw_shard_id
+      set_fact:
+       mw_shard_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+  roles:
+    - learning-service
+  serial: 1

lp_learning_neo4j_deploy.yml

@@ -0,0 +1,4 @@
+- hosts: learning-neo4j-cluster
+  roles: 
+    - neo4j-deploy
+

lp_learning_neo4j_provision.yml

@@ -0,0 +1,25 @@
+- hosts: learning-neo4j-cluster
+  pre_tasks:
+    - name: Create learner group
+      become: yes
+      group:
+        state: present
+        name: "{{ learner_group }}"
+        system: yes
+    - name: Create learner user
+      become: yes
+      user:
+        state: present
+        name: "{{ learner_user }}"
+        comment: learner user
+        system: yes
+        createhome: yes
+        group: "{{ learner_group }}"
+    - name: Registering server_id
+      set_fact:
+        server_id: "{% for servername in play_hosts %}{% if inventory_hostname==servername %}{{ loop.index }}{% endif %}{% endfor %}"
+  roles: 
+    - java
+    - neo4j-cluster
+    - logstash
+

lp_learning_provision.yml

@@ -0,0 +1,27 @@
+- hosts: learningall
+  pre_tasks:
+    - name: Create learner group
+      become: yes
+      group:
+        state: present
+        name: "{{ learner_group }}"
+        system: yes
+    - name: Create learner user
+      become: yes
+      user:
+        state: present
+        name: "{{ learner_user }}"
+        comment: learner user
+        system: yes
+        createhome: yes
+        group: "{{ learner_group }}"
+    - name: create folder data for learning
+      become: yes
+      file: path={{ item }} state=directory mode=0775 owner={{learner_user}} group={{learner_user}}
+      with_items: "{{ learnings.paths }}"
+  roles:
+    - java
+    - tomcat
+    - ffmpeg
+    - imagemagick
+    - logstash

lp_learning_validation.yml

@@ -0,0 +1,6 @@
+- hosts: learningall
+  vars_files:
+       - "{{ secrets_path }}"
+  roles:
+    - lp-learning-validation
+  serial: 1

lp_logstash_deploy.yml

@@ -0,0 +1,5 @@
+- hosts: "{{ remote }}"
+  roles:
+    - logstash-deploy
+  serial: 1
+

lp_neo4j-backup.yml

@@ -0,0 +1,4 @@
+- hosts: learning-neo4j-node1
+  remote_user: learning
+  roles:
+        - neo4j-backup 

lp_redis_provision.yml

@@ -0,0 +1,5 @@
+- hosts: redisall
+  roles:
+    - redis
+  serial: 1
+

lp_samza_deploy.yml

@@ -0,0 +1,31 @@
+---
+- name: "Start Nodemanager on Slaves"
+  hosts: "yarn-slave"
+  become: yes
+  tasks:
+    - name: Ensure yarn nodemanager is running
+      become_user: hduser
+      shell: |
+         (ps aux | grep yarn-hduser-nodemanager | grep -v grep) || /usr/local/hadoop/sbin/yarn-daemon.sh --config /usr/local/hadoop-{{hadoop_version}}/conf/ start nodemanager || sleep 10
+
+    - name: Install mysql client
+      apt: name=mysql-client state=present
+
+    - name: install imagemagick
+      apt: name=imagemagick state=present update_cache=yes
+
+- name: "Copy Samza jobs additional configuration to slaves"
+  become: yes
+  hosts: "yarn-slave"
+  roles:
+    - samza-jobs-additional-config
+
+- name: "Deploy Samza jobs"
+  hosts: "yarn-master"
+  become: yes
+  vars_files:
+    - "{{ secrets_path }}"
+  vars:
+    deploy_jobs: true
+  roles:
+    - samza-jobs

lp_samza_telemetry_schemas.yml

@@ -0,0 +1,26 @@
+---
+- name: "Copy validation schema files to Yarn Slaves"
+  hosts: "yarn-slave"
+  become: yes
+  tasks:
+   - name: cloning the telemetry schema repo
+     git:
+       repo: "{{schema_repo_url}}"
+       dest: "{{telemetry_schema_directory}}"
+       version: "{{version}}"
+       force: yes
+   - name: Create schema directory
+     file: path={{telemetry_schema_directory}} owner=hduser group=hadoop recurse=yes state=directory
+
+   - name: get schema dir names
+     raw: find {{telemetry_schema_path}} -type f -name "*.*"
+     register: schemas
+
+   - name: change internal schema file reference
+     replace: 
+       dest: "{{item}}" 
+       regexp: "http://localhost:7070/schemas/" 
+       replace: "file://{{telemetry_schema_path}}/" 
+       owner: hduser 
+       group: hadoop
+     with_items: "{{ schemas.stdout_lines }}"

lp_search_deploy.yml

@@ -0,0 +1,4 @@
+- hosts: searchall
+  roles:
+    - search-service-deploy
+  serial: 1

lp_search_provision.yml

@@ -0,0 +1,26 @@
+- hosts: searchall
+  pre_tasks:
+    - name: Create learner group
+      become: yes
+      group:
+        state: present
+        name: "{{ learner_group }}"
+        system: yes
+    - name: Create learner user
+      become: yes
+      user:
+        state: present
+        name: "{{ learner_user }}"
+        comment: learner user
+        system: yes
+        createhome: yes
+        group: "{{ learner_group }}"
+    - name: create folder data for learning
+      become: yes
+      file: path={{ item }} state=directory mode=0755 owner={{learner_user}} group={{learner_group}}
+      with_items: "{{ searchs.paths }}"
+  roles:
+    - java
+    - logstash
+    - search-service
+  serial: 1

lp_start_neo4j.yml

@@ -0,0 +1,15 @@
+- hosts: learning-neo4j-cluster
+  tasks:
+    - name: Check if neo4j is running
+      become_user: "{{ learner_user }}"
+      shell: ps -ef | grep "{{ neo4j_home }}"  | grep -v grep | wc -l
+      register: neo4j_process
+
+    - name: Start neo4j
+      become: yes
+      become_user: "{{ learner_user }}"
+      shell:  bin/neo4j start
+      args:
+        chdir: "{{ neo4j_home }}"
+      when: "neo4j_process.stdout == '0'"
+  serial: 1

lp_stop_neo4j.yml

@@ -0,0 +1,15 @@
+- hosts: learning-neo4j-cluster
+  tasks:
+    - name: Check if neo4j is running
+      become_user: "{{ learner_user }}"
+      shell: ps -ef | grep "{{ neo4j_home }}"  | grep -v grep | wc -l
+      register: neo4j_process
+
+    - name: stop the neo4j
+      become: yes
+      become_user: "{{ learner_user }}"
+      shell:  bin/neo4j stop
+      args:
+        chdir: "{{ neo4j_home }}"
+      when: "neo4j_process.stdout != '0'"
+  serial: 1

lp_synctool_deploy.yml

@@ -0,0 +1,5 @@
+- hosts: learning1
+  vars_files:
+    - "{{ secrets_path }}"
+  roles:
+    - lp-synctool-deploy