diff --git a/VERSION b/VERSION
index edcfe40d..4a36342f 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.14.0
+3.0.0
diff --git a/resources/bin/rb_check b/resources/bin/rb_check
index f1f7a8e9..ab124a39 100755
--- a/resources/bin/rb_check
+++ b/resources/bin/rb_check
@@ -28,7 +28,6 @@ fi
EXITRET=0
RES_COL=120
-SAMZA=0
MOVE_TO_COL="echo -en \\033[${RES_COL}G"
SCRIPTS_PATH="/etc/rb_check.d"
@@ -42,9 +41,6 @@ extended=0
waitfinish=0
s=""
-grep -q samza /etc/topicmode/*
-[ $? -eq 0 ] && SAMZA=1
-
# Auxiliar functions to check if exists another rb_check process launched
LOCKFILE="/tmp/rb_check.lock"
function f_trap_exit() {
@@ -358,14 +354,6 @@ e_title " DATE: $(date) "
[ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xhistorical" -o "x$s" == "xdruid_historical" ] && check_druid_historical
[ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xbroker" -o "x$s" == "xdruid_broker" ] && check_druid_broker
[ -f ${SCRIPTS_PATH}/*webui* ] && [ "x$s" == "x" -o "x$s" == "xrb_webui" -o "x$s" == "xwebui" -o "x$s" == "xrb-webui" ] && check_webui
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xresourcemanager" -o "x$s" == "xhadoop_resourcemanager" ] && check_hadoop_resourcemanager
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnodemanager" -o "x$s" == "xhadoop_nodemanager" ] && check_hadoop_nodemanager
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnamenode" -o "x$s" == "xhadoop_namenode" ] && check_hadoop_namenode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xjournalnode" -o "x$s" == "xhadoop_journalnode" ] && check_hadoop_journalnode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xdatanode" -o "x$s" == "xhadoop_datanode" ] && check_hadoop_datanode
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhistoryserver" -o "x$s" == "xhadoop_historyserver" ] && check_hadoop_historyserver
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xzkfc" -o "x$s" == "xhadoop_zkfc" ] && check_hadoop_zkfc
-[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhdfs" -o "x$s" == "xhadoop_hdfs" ] && check_hadoop_hdfs
[ -f ${SCRIPTS_PATH}/*keepalived* ] && [ "x$s" == "x" -o "x$s" == "xkeepalived" ] && check_keepalived
[ -f ${SCRIPTS_PATH}/*aerospike* ] && [ "x$s" == "x" -o "x$s" == "xaerospike" ] && check_aerospike
[ -f ${SCRIPTS_PATH}/*drill* ] && [ "x$s" == "x" -o "x$s" == "xdrill" ] && check_drill
@@ -384,10 +372,6 @@ if [ $extended -eq 1 ]; then
e_title "Extended INFO: rb_get_topics.rb"
eval $CMDBEGIN /usr/lib/redborder/scripts/rb_get_topics.rb
- if [ $SAMZA -eq 1 ]; then
- e_title "Extended INFO: rb_get_topics.rb"
- timeout 30 /usr/lib/redborder/bin/rb_samza.sh -l
- fi
fi
if [ "x$upload" != "x" ]; then
diff --git a/resources/bin/rb_configure_custom.sh b/resources/bin/rb_configure_custom.sh
index 307fc96f..451880b4 100755
--- a/resources/bin/rb_configure_custom.sh
+++ b/resources/bin/rb_configure_custom.sh
@@ -125,6 +125,10 @@ chmod a+r /etc/chef/client.pem
#Add client to admins group
knife group add client `hostname -s` admins &>/dev/null
+e_title "Enabling chef-client service"
+systemctl enable chef-client
+systemctl start chef-client
+
# Copy dhclient hook
cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks
diff --git a/resources/bin/rb_configure_leader.sh b/resources/bin/rb_configure_leader.sh
index 2562fa83..0ce98f45 100755
--- a/resources/bin/rb_configure_leader.sh
+++ b/resources/bin/rb_configure_leader.sh
@@ -409,7 +409,7 @@ function configure_leader(){
# Multiple runs of chef-client
e_title "Configuring Chef-Client. Please wait... "
- e_title "redborder install run (1/4) $(date)" | tee -a /root/.install-chef-client.log
+ e_title "redborder install run (1/2) $(date)" | tee -a /root/.install-chef-client.log
chef-client | tee -a /root/.install-chef-client.log
# Replace chef-server SV init scripts by systemd scripts
@@ -424,16 +424,7 @@ function configure_leader(){
done
fi
- e_title "redborder install run (2/4) $(date)" | tee -a /root/.install-chef-client.log
- chef-client | tee -a /root/.install-chef-client.log
-
- e_title "redborder install run (3/4) $(date)" | tee -a /root/.install-chef-client.log
- chef-client | tee -a /root/.install-chef-client.log
-
- e_title "Creating database structure $(date)"
- chef-solo -c /var/chef/solo/webui-solo.rb -j /var/chef/solo/webui-attributes.json
-
- e_title "redborder install run (4/4) $(date)" | tee -a /root/.install-chef-client.log
+ e_title "redborder install run (2/2) $(date)" | tee -a /root/.install-chef-client.log
chef-client | tee -a /root/.install-chef-client.log
}
@@ -543,8 +534,17 @@ configure_leader
#rm -f /etc/opscode/chef-server.rb
rm -f /var/lock/leader-configuring.lock
+e_title "Enabling chef-client service"
+systemctl enable chef-client
+systemctl start chef-client
+
+e_title "Starting default services"
+for service in logstash webui rb-workers sfacctd f2k redborder-mem2incident; do
+ [ "$(systemctl is-enabled $service 2>/dev/null)" = "enabled" ] && systemctl start $service &>/dev/null &
+done
+
# Configure default druid rule (load 1 month, drop forever)
-e_title "Configuring default druid rule"
+e_title "Configuring default 1 month data retention"
/usr/lib/redborder/bin/rb_druid_rules -t _default -p none -d p1m -i 1
# Copy dhclient hook
@@ -552,7 +552,7 @@ cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks
e_title "Configuring cgroups (first time), please wait..."
-rb_configure_cgroups &>/dev/null
+/usr/lib/redborder/bin/rb_configure_cgroups
echo "Cgroups configured in /sys/fs/cgroup/redborder.slice/"
diff --git a/resources/bin/rb_refresh_darklist_memcached_keys.sh b/resources/bin/rb_refresh_darklist_memcached_keys.sh
index e4be48e5..849621fe 100644
--- a/resources/bin/rb_refresh_darklist_memcached_keys.sh
+++ b/resources/bin/rb_refresh_darklist_memcached_keys.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza
+# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash
source /etc/profile.d/rvm.sh
diff --git a/resources/bin/rb_samza.sh b/resources/bin/rb_samza.sh
deleted file mode 100755
index e0353db2..00000000
--- a/resources/bin/rb_samza.sh
+++ /dev/null
@@ -1,187 +0,0 @@
-#!/bin/bash
-
-#######################################################################
-# Copyright (c) 2014 ENEO Tecnología S.L.
-# This file is part of redBorder.
-# redBorder is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-# redBorder is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License License for more details.
-# You should have received a copy of the GNU Affero General Public License License
-# along with redBorder. If not, see .
-#######################################################################
-
-#RBDIR=${RBDIR-/opt/rb}
-source /etc/profile
-
-function usage(){
- echo "$0 [-h][-e][-u][-k][-l][-c ][-m ]"
- echo " -h: print this help"
- echo " -e: execute the task"
- echo " -v: print the version"
- echo " -k: kill running tasks"
- echo " -l: list running tasks"
- echo " -t: set type of task: enrichment, indexing, pms, all (default)"
- echo " -c: set samza containers (default: 1)"
- echo " -m: set samza container memory (default: 1024)"
- echo " -s: show samza container info"
- echo " example:"
- echo " - upload: rb_samza.sh -t enrichment -e"
- echo " - kill: rb_samza.sh -t enrichment -k"
- echo " - list: rb_samza.sh -l"
- exit 1
-}
-
-f_execute=0
-f_kill=0
-f_list=0
-f_cont=0
-f_mem=0
-containers=1
-memory=1024
-f_version=0
-f_type="all"
-f_show=0
-
-while getopts "hevklt:c:m:s" name
-do
- case $name in
- h) usage;;
- e) f_execute=1;;
- k) f_kill=1;;
- l) f_list=1;;
- v) f_version=1;;
- t) f_type=$OPTARG;;
- c) f_cont=1; containers=$OPTARG;;
- m) f_mem=1; memory=$OPTARG;;
- s) f_show=1;;
- esac
-done
-
-if [ $f_version -eq 1 ]; then
- ls /var/rb-samza-bi/app/rb-samza-bi.tar.gz -l|sed 's/.*-> //'|awk '{print $1}' | sed 's/rb-samza-bi-//' | sed 's/-.*$//'
-else
- [ $f_cont -eq 1 ] && rb_set_samzacontainers.rb -c $containers
- [ $f_mem -eq 1 ] && rb_set_samzacontainersmemory.rb -m $memory
- [ $f_show -eq 1 ] && rb_set_samzacontainers.rb -s && rb_set_samzacontainersmemory.rb -s
- if [ $f_kill -eq 1 ]; then
- if [ "x$f_type" == "xenrichment" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep enrichment | awk '{print $1}')"
- running="x$application";
- if [ $running != "x" ]; then
- yarn application -kill $application &>/dev/null
- echo "Killed application $application (enrichment)"
- else
- echo "The enrichment application was not running"
- fi
- fi
-
- if [ "x$f_type" == "xlocation" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep location | awk '{print $1}')"
- running="x$application";
- if [ $running != "x" ]; then
- yarn application -kill $application &>/dev/null
- echo "Killed application $application (location)"
- else
- echo "The location application was not running"
- fi
- fi
-
- if [ "x$f_type" == "xindexing" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep indexing | awk '{print $1}')"
- running="x$application";
- if [ $running != "x" ]; then
- yarn application -kill $application &>/dev/null
- echo "Killed application $application (indexing)"
- else
- echo "The indexing application was not running"
- fi
- fi
-
- if [ "x$f_type" == "xmalware" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep samza-malware | awk '{print $1}')"
- running="x$application";
- if [ $running != "x" ]; then
- yarn application -kill $application &>/dev/null
- echo "Killed application $application (malware)"
- else
- echo "The malware application was not running"
- fi
- fi
-
- if [ "x$f_type" == "xpms" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep pms | awk '{print $1}')"
- running="x$application";
- if [ $running != "x" ]; then
- yarn application -kill $application &>/dev/null
- echo "Killed application $application (pms)"
- else
- echo "The pms application was not running"
- fi
- fi
- fi
-
- if [ $f_list -eq 1 ]; then
- timeout 60 yarn application -list | egrep "(Samza|Application-Id)"
- fi
-
- if [ $f_execute -eq 1 ]; then
- if [ "x$f_type" == "xmalware" ]; then
-
- rm -rf /var/rb-samza-malware/bin/*
- rm -rf /var/rb-samza-malware/lib/*
- tar xfz /var/rb-samza-malware/app/rb-samza-malware.tar.gz -C /var/rb-samza-malware
-
- application="$(yarn application -list | grep samza-malware | awk '{print $1}')"
- running="x$application";
- if [ $running == "x" ]; then
- /var/rb-samza-malware/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-malware/config/malware.properties
- else
- echo "Application $application (malware) is already running."
- fi
- else
- rm -rf /var/rb-samza-bi/bin/*
- rm -rf /var/rb-samza-bi/lib/*
- tar xfz /var/rb-samza-bi/app/rb-samza-bi.tar.gz -C /var/rb-samza-bi
-
- if [ "x$f_type" == "xenrichment" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep enrichment | awk '{print $1}')"
- running="x$application";
- if [ $running == "x" ]; then
- /var/rb-samza-bi/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-bi/config/enrichment.properties
- else
- echo "Application $application (enrichment) is already running."
- fi
- fi
-
- if [ "x$f_type" == "xlocation" ] || [ "x$f_type" == "xall" ]; then
- rm -rf /var/rb-samza-location/bin/*
- rm -rf /var/rb-samza-location/lib/*
- tar xfz /var/rb-samza-location/app/rb-samza-location.tar.gz -C /var/rb-samza-location
-
- application="$(yarn application -list | grep location | awk '{print $1}')"
- running="x$application";
- if [ $running == "x" ]; then
- /var/rb-samza-location/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-location/config/location.properties
- else
- echo "Application $application (location) is already running."
- fi
- fi
-
- if [ "x$f_type" == "xindexing" ] || [ "x$f_type" == "xall" ]; then
- application="$(yarn application -list | grep indexing | awk '{print $1}')"
- running="x$application";
- if [ $running == "x" ]; then
- /var/rb-samza-bi/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-bi/config/indexing.properties
- else
- echo "Application $application (indexing) is already running."
- fi
- fi
- fi
- fi
-
-fi
diff --git a/resources/bin/rb_update_darklist.sh b/resources/bin/rb_update_darklist.sh
index c6b6baf0..9e8626e1 100644
--- a/resources/bin/rb_update_darklist.sh
+++ b/resources/bin/rb_update_darklist.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza
+# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash
source /etc/profile.d/rvm.sh
diff --git a/resources/bin/rb_upload_cookbooks.sh b/resources/bin/rb_upload_cookbooks.sh
index f960b48c..bcb0cdcd 100755
--- a/resources/bin/rb_upload_cookbooks.sh
+++ b/resources/bin/rb_upload_cookbooks.sh
@@ -48,7 +48,7 @@ function upload_cookbook() {
listCookbooks="rb-common rb-selinux cron zookeeper kafka druid http2k memcached chef-server consul
- hadoop samza nginx geoip webui snmp mongodb rbmonitor rbscanner
+ nginx geoip webui snmp mongodb rbmonitor rbscanner
f2k logstash pmacct minio postgresql rbdswatcher rbevents-counter
rsyslog freeradius rbnmsp n2klocd rbale rbcep k2http rblogstatter rb-arubacentral rbcgroup rb-exporter rb-chrony rb-clamav rb-postfix
keepalived snort barnyard2 rbaioutliers mem2incident rb-ai
diff --git a/resources/chef/solo/webui-attributes.json b/resources/chef/solo/webui-attributes.json
index 9b26918d..3ed9f974 100644
--- a/resources/chef/solo/webui-attributes.json
+++ b/resources/chef/solo/webui-attributes.json
@@ -1,6 +1,5 @@
{
"run_list" : [
- "recipe[webui::configure_db]",
"recipe[webui::configure_modules]"
]
}
diff --git a/resources/etc/mode-list.yml b/resources/etc/mode-list.yml
index fc055e21..18608aa1 100644
--- a/resources/etc/mode-list.yml
+++ b/resources/etc/mode-list.yml
@@ -28,9 +28,6 @@
-
name: 'http2k'
description: 'Http2k node'
--
- name: 'samza'
- description: 'Samza node'
-
name: 'f2k'
description: 'f2k flow collector node'
diff --git a/resources/scripts/rb_backup_node.rb b/resources/scripts/rb_backup_node.rb
index 4a910959..06d87153 100755
--- a/resources/scripts/rb_backup_node.rb
+++ b/resources/scripts/rb_backup_node.rb
@@ -328,7 +328,7 @@ def check_oper (cmd, mode, type, message)
end
# Stop chef-client
- check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka hadoop_ stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ")
+ check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ")
# Restore the node
check_oper("#{tar} #{path} -C /", verbose, type, "Restoring files ... ")
`sed -i '/rb_aws_secondary_ip.sh/d' /etc/keepalived/keepalived.conf`
@@ -430,7 +430,7 @@ def check_oper (cmd, mode, type, message)
end
# empty virtual ips on new environment
- ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-hadoop_namenode", "ipvirtual-internal-hadoop_resourcemanager", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x|
+ ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x|
db_temp = Chef::DataBagItem.load('rBglobal', x ) rescue db_temp = nil
if !db_temp.nil? and !db_temp["ip"].nil? and db_temp["ip"]!=""
db_temp["ip"] = ""
diff --git a/resources/scripts/rb_get_topics.rb b/resources/scripts/rb_get_topics.rb
index 5b2ab85c..42307bd6 100755
--- a/resources/scripts/rb_get_topics.rb
+++ b/resources/scripts/rb_get_topics.rb
@@ -98,7 +98,7 @@ def get_partitions(zk, topic)
topics.each do |topic|
next if topic=="rb_alarm" or topic=="__consumer_offsets" or topic=="app"
- #next if topic.start_with?"__samza_"
+
index=0
if zk.exists?("#{topic_path(topic)}")
partitions = zk.children(partitions_path(topic)).map{|k| k.to_i}.sort.uniq