Merge pull request #186 from redBorder/development

Release 3.0.0

VERSION

@@ -1 +1 @@
-2.14.0
+3.0.0

resources/bin/rb_check

@@ -28,7 +28,6 @@ fi
 
 EXITRET=0
 RES_COL=120
-SAMZA=0
 MOVE_TO_COL="echo -en \\033[${RES_COL}G"
 SCRIPTS_PATH="/etc/rb_check.d"
 
@@ -42,9 +41,6 @@ extended=0
 waitfinish=0
 s=""
 
-grep -q samza /etc/topicmode/*
-[ $? -eq 0 ] && SAMZA=1
-
 # Auxiliar functions to check if exists another rb_check process launched
 LOCKFILE="/tmp/rb_check.lock"
 function f_trap_exit() {   
@@ -358,14 +354,6 @@ e_title "          DATE: $(date)    "
 [ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xhistorical" -o "x$s" == "xdruid_historical" ] && check_druid_historical
 [ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xbroker" -o "x$s" == "xdruid_broker" ] && check_druid_broker
 [ -f ${SCRIPTS_PATH}/*webui* ] && [ "x$s" == "x" -o "x$s" == "xrb_webui" -o "x$s" == "xwebui" -o "x$s" == "xrb-webui" ] && check_webui
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xresourcemanager" -o "x$s" == "xhadoop_resourcemanager" ] && check_hadoop_resourcemanager
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnodemanager" -o "x$s" == "xhadoop_nodemanager" ] && check_hadoop_nodemanager
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnamenode" -o "x$s" == "xhadoop_namenode" ] && check_hadoop_namenode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xjournalnode" -o "x$s" == "xhadoop_journalnode" ] && check_hadoop_journalnode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xdatanode" -o "x$s" == "xhadoop_datanode" ] && check_hadoop_datanode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhistoryserver" -o "x$s" == "xhadoop_historyserver" ] && check_hadoop_historyserver
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xzkfc" -o "x$s" == "xhadoop_zkfc" ] && check_hadoop_zkfc
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhdfs" -o "x$s" == "xhadoop_hdfs" ] && check_hadoop_hdfs 
 [ -f ${SCRIPTS_PATH}/*keepalived* ] && [ "x$s" == "x" -o "x$s" == "xkeepalived" ] && check_keepalived
 [ -f ${SCRIPTS_PATH}/*aerospike* ] && [ "x$s" == "x" -o "x$s" == "xaerospike" ] && check_aerospike
 [ -f ${SCRIPTS_PATH}/*drill* ] && [ "x$s" == "x" -o "x$s" == "xdrill" ] && check_drill
@@ -384,10 +372,6 @@ if [ $extended -eq 1 ]; then
   e_title "Extended INFO: rb_get_topics.rb"
   eval $CMDBEGIN /usr/lib/redborder/scripts/rb_get_topics.rb
 
-  if [ $SAMZA -eq 1 ]; then
-    e_title "Extended INFO: rb_get_topics.rb"
-    timeout 30 /usr/lib/redborder/bin/rb_samza.sh -l
-  fi
 fi
 
 if [ "x$upload" != "x" ]; then

resources/bin/rb_configure_custom.sh

@@ -125,6 +125,10 @@ chmod a+r /etc/chef/client.pem
 #Add client to admins group
 knife group add client `hostname -s` admins &>/dev/null
 
+e_title "Enabling chef-client service"
+systemctl enable chef-client
+systemctl start chef-client
+
 # Copy dhclient hook
 cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks
 

resources/bin/rb_configure_leader.sh

@@ -409,7 +409,7 @@ function configure_leader(){
   # Multiple runs of chef-client
   e_title "Configuring Chef-Client. Please wait...  "
 
-  e_title "redborder install run (1/4) $(date)" | tee -a /root/.install-chef-client.log
+  e_title "redborder install run (1/2) $(date)" | tee -a /root/.install-chef-client.log
   chef-client | tee -a /root/.install-chef-client.log
 
   # Replace chef-server SV init scripts by systemd scripts
@@ -424,16 +424,7 @@ function configure_leader(){
     done
   fi
 
-  e_title "redborder install run (2/4) $(date)" | tee -a /root/.install-chef-client.log
-  chef-client | tee -a /root/.install-chef-client.log
-
-  e_title "redborder install run (3/4) $(date)" | tee -a /root/.install-chef-client.log
-  chef-client | tee -a /root/.install-chef-client.log
-
-  e_title "Creating database structure $(date)"
-  chef-solo -c /var/chef/solo/webui-solo.rb -j /var/chef/solo/webui-attributes.json
-
-  e_title "redborder install run (4/4) $(date)" | tee -a /root/.install-chef-client.log
+  e_title "redborder install run (2/2) $(date)" | tee -a /root/.install-chef-client.log
   chef-client | tee -a /root/.install-chef-client.log
 }
 
@@ -543,16 +534,25 @@ configure_leader
 #rm -f /etc/opscode/chef-server.rb
 rm -f /var/lock/leader-configuring.lock
 
+e_title "Enabling chef-client service"
+systemctl enable chef-client
+systemctl start chef-client
+
+e_title "Starting default services"
+for service in logstash webui rb-workers sfacctd f2k redborder-mem2incident; do
+  [ "$(systemctl is-enabled $service 2>/dev/null)" = "enabled" ] && systemctl start $service &>/dev/null &
+done
+
 # Configure default druid rule (load 1 month, drop forever)
-e_title "Configuring default druid rule"
+e_title "Configuring default 1 month data retention"
 /usr/lib/redborder/bin/rb_druid_rules -t _default -p none -d p1m -i 1
 
 # Copy dhclient hook
 cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks
 
 e_title "Configuring cgroups (first time), please wait..."
 
-rb_configure_cgroups &>/dev/null
+/usr/lib/redborder/bin/rb_configure_cgroups
 
 echo "Cgroups configured in /sys/fs/cgroup/redborder.slice/"
 

resources/bin/rb_refresh_darklist_memcached_keys.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza
+# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash
 
 source /etc/profile.d/rvm.sh
 

resources/bin/rb_update_darklist.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza
+# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash
 
 source /etc/profile.d/rvm.sh
 

resources/bin/rb_upload_cookbooks.sh

@@ -48,7 +48,7 @@ function upload_cookbook() {
 
 
 listCookbooks="rb-common rb-selinux cron zookeeper kafka druid http2k memcached chef-server consul
-               hadoop samza nginx geoip webui snmp mongodb rbmonitor rbscanner
+               nginx geoip webui snmp mongodb rbmonitor rbscanner
                f2k logstash pmacct minio postgresql rbdswatcher rbevents-counter
                rsyslog freeradius rbnmsp n2klocd rbale rbcep k2http rblogstatter rb-arubacentral rbcgroup rb-exporter rb-chrony rb-clamav rb-postfix
                keepalived snort barnyard2 rbaioutliers mem2incident rb-ai

resources/chef/solo/webui-attributes.json

@@ -1,6 +1,5 @@
 {
   "run_list" : [
-    "recipe[webui::configure_db]",
     "recipe[webui::configure_modules]"
   ]
 }

resources/etc/mode-list.yml

@@ -28,9 +28,6 @@
 -
   name: 'http2k'
   description: 'Http2k node'
--
-  name: 'samza'
-  description: 'Samza node'
 -
   name: 'f2k'
   description: 'f2k flow collector node'

resources/scripts/rb_backup_node.rb

@@ -328,7 +328,7 @@ def check_oper (cmd, mode, type, message)
     end
 
     # Stop chef-client
-    check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka hadoop_ stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ")
+    check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ")
     # Restore the node
     check_oper("#{tar} #{path} -C /", verbose, type, "Restoring files ... ")
     `sed -i '/rb_aws_secondary_ip.sh/d' /etc/keepalived/keepalived.conf`
@@ -430,7 +430,7 @@ def check_oper (cmd, mode, type, message)
       end
 
       # empty virtual ips on new environment
-      ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-hadoop_namenode", "ipvirtual-internal-hadoop_resourcemanager", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x|
+      ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x|
         db_temp = Chef::DataBagItem.load('rBglobal', x ) rescue db_temp = nil
         if !db_temp.nil? and !db_temp["ip"].nil? and db_temp["ip"]!=""
           db_temp["ip"] = ""

resources/scripts/rb_get_topics.rb

@@ -98,7 +98,7 @@ def get_partitions(zk, topic)
 
     topics.each do |topic|
       next if topic=="rb_alarm" or topic=="__consumer_offsets" or topic=="app"
-      #next if topic.start_with?"__samza_"
+
       index=0
       if zk.exists?("#{topic_path(topic)}")
         partitions = zk.children(partitions_path(topic)).map{|k| k.to_i}.sort.uniq