diff --git a/VERSION b/VERSION index edcfe40d..4a36342f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.14.0 +3.0.0 diff --git a/resources/bin/rb_check b/resources/bin/rb_check index f1f7a8e9..ab124a39 100755 --- a/resources/bin/rb_check +++ b/resources/bin/rb_check @@ -28,7 +28,6 @@ fi EXITRET=0 RES_COL=120 -SAMZA=0 MOVE_TO_COL="echo -en \\033[${RES_COL}G" SCRIPTS_PATH="/etc/rb_check.d" @@ -42,9 +41,6 @@ extended=0 waitfinish=0 s="" -grep -q samza /etc/topicmode/* -[ $? -eq 0 ] && SAMZA=1 - # Auxiliar functions to check if exists another rb_check process launched LOCKFILE="/tmp/rb_check.lock" function f_trap_exit() { @@ -358,14 +354,6 @@ e_title " DATE: $(date) " [ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xhistorical" -o "x$s" == "xdruid_historical" ] && check_druid_historical [ -f ${SCRIPTS_PATH}/*druid* ] && [ "x$s" == "x" -o "x$s" == "xdruid" -o "x$s" == "xbroker" -o "x$s" == "xdruid_broker" ] && check_druid_broker [ -f ${SCRIPTS_PATH}/*webui* ] && [ "x$s" == "x" -o "x$s" == "xrb_webui" -o "x$s" == "xwebui" -o "x$s" == "xrb-webui" ] && check_webui -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xresourcemanager" -o "x$s" == "xhadoop_resourcemanager" ] && check_hadoop_resourcemanager -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnodemanager" -o "x$s" == "xhadoop_nodemanager" ] && check_hadoop_nodemanager -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xnamenode" -o "x$s" == "xhadoop_namenode" ] && check_hadoop_namenode -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xjournalnode" -o "x$s" == "xhadoop_journalnode" ] && check_hadoop_journalnode -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xdatanode" -o "x$s" == "xhadoop_datanode" ] && check_hadoop_datanode -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhistoryserver" -o "x$s" == "xhadoop_historyserver" ] && check_hadoop_historyserver -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xzkfc" -o "x$s" == "xhadoop_zkfc" ] && check_hadoop_zkfc -[ -f ${SCRIPTS_PATH}/*hadoop* ] && [ "x$s" == "x" -o "x$s" == "xhadoop" -o "x$s" == "xhdfs" -o "x$s" == "xhadoop_hdfs" ] && check_hadoop_hdfs [ -f ${SCRIPTS_PATH}/*keepalived* ] && [ "x$s" == "x" -o "x$s" == "xkeepalived" ] && check_keepalived [ -f ${SCRIPTS_PATH}/*aerospike* ] && [ "x$s" == "x" -o "x$s" == "xaerospike" ] && check_aerospike [ -f ${SCRIPTS_PATH}/*drill* ] && [ "x$s" == "x" -o "x$s" == "xdrill" ] && check_drill @@ -384,10 +372,6 @@ if [ $extended -eq 1 ]; then e_title "Extended INFO: rb_get_topics.rb" eval $CMDBEGIN /usr/lib/redborder/scripts/rb_get_topics.rb - if [ $SAMZA -eq 1 ]; then - e_title "Extended INFO: rb_get_topics.rb" - timeout 30 /usr/lib/redborder/bin/rb_samza.sh -l - fi fi if [ "x$upload" != "x" ]; then diff --git a/resources/bin/rb_configure_custom.sh b/resources/bin/rb_configure_custom.sh index 307fc96f..451880b4 100755 --- a/resources/bin/rb_configure_custom.sh +++ b/resources/bin/rb_configure_custom.sh @@ -125,6 +125,10 @@ chmod a+r /etc/chef/client.pem #Add client to admins group knife group add client `hostname -s` admins &>/dev/null +e_title "Enabling chef-client service" +systemctl enable chef-client +systemctl start chef-client + # Copy dhclient hook cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks diff --git a/resources/bin/rb_configure_leader.sh b/resources/bin/rb_configure_leader.sh index 2562fa83..0ce98f45 100755 --- a/resources/bin/rb_configure_leader.sh +++ b/resources/bin/rb_configure_leader.sh @@ -409,7 +409,7 @@ function configure_leader(){ # Multiple runs of chef-client e_title "Configuring Chef-Client. Please wait... " - e_title "redborder install run (1/4) $(date)" | tee -a /root/.install-chef-client.log + e_title "redborder install run (1/2) $(date)" | tee -a /root/.install-chef-client.log chef-client | tee -a /root/.install-chef-client.log # Replace chef-server SV init scripts by systemd scripts @@ -424,16 +424,7 @@ function configure_leader(){ done fi - e_title "redborder install run (2/4) $(date)" | tee -a /root/.install-chef-client.log - chef-client | tee -a /root/.install-chef-client.log - - e_title "redborder install run (3/4) $(date)" | tee -a /root/.install-chef-client.log - chef-client | tee -a /root/.install-chef-client.log - - e_title "Creating database structure $(date)" - chef-solo -c /var/chef/solo/webui-solo.rb -j /var/chef/solo/webui-attributes.json - - e_title "redborder install run (4/4) $(date)" | tee -a /root/.install-chef-client.log + e_title "redborder install run (2/2) $(date)" | tee -a /root/.install-chef-client.log chef-client | tee -a /root/.install-chef-client.log } @@ -543,8 +534,17 @@ configure_leader #rm -f /etc/opscode/chef-server.rb rm -f /var/lock/leader-configuring.lock +e_title "Enabling chef-client service" +systemctl enable chef-client +systemctl start chef-client + +e_title "Starting default services" +for service in logstash webui rb-workers sfacctd f2k redborder-mem2incident; do + [ "$(systemctl is-enabled $service 2>/dev/null)" = "enabled" ] && systemctl start $service &>/dev/null & +done + # Configure default druid rule (load 1 month, drop forever) -e_title "Configuring default druid rule" +e_title "Configuring default 1 month data retention" /usr/lib/redborder/bin/rb_druid_rules -t _default -p none -d p1m -i 1 # Copy dhclient hook @@ -552,7 +552,7 @@ cp -f /usr/lib/redborder/lib/dhclient-enter-hooks /etc/dhcp/dhclient-enter-hooks e_title "Configuring cgroups (first time), please wait..." -rb_configure_cgroups &>/dev/null +/usr/lib/redborder/bin/rb_configure_cgroups echo "Cgroups configured in /sys/fs/cgroup/redborder.slice/" diff --git a/resources/bin/rb_refresh_darklist_memcached_keys.sh b/resources/bin/rb_refresh_darklist_memcached_keys.sh index e4be48e5..849621fe 100644 --- a/resources/bin/rb_refresh_darklist_memcached_keys.sh +++ b/resources/bin/rb_refresh_darklist_memcached_keys.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza +# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash source /etc/profile.d/rvm.sh diff --git a/resources/bin/rb_samza.sh b/resources/bin/rb_samza.sh deleted file mode 100755 index e0353db2..00000000 --- a/resources/bin/rb_samza.sh +++ /dev/null @@ -1,187 +0,0 @@ -#!/bin/bash - -####################################################################### -# Copyright (c) 2014 ENEO Tecnología S.L. -# This file is part of redBorder. -# redBorder is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# redBorder is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License License for more details. -# You should have received a copy of the GNU Affero General Public License License -# along with redBorder. If not, see . -####################################################################### - -#RBDIR=${RBDIR-/opt/rb} -source /etc/profile - -function usage(){ - echo "$0 [-h][-e][-u][-k][-l][-c ][-m ]" - echo " -h: print this help" - echo " -e: execute the task" - echo " -v: print the version" - echo " -k: kill running tasks" - echo " -l: list running tasks" - echo " -t: set type of task: enrichment, indexing, pms, all (default)" - echo " -c: set samza containers (default: 1)" - echo " -m: set samza container memory (default: 1024)" - echo " -s: show samza container info" - echo " example:" - echo " - upload: rb_samza.sh -t enrichment -e" - echo " - kill: rb_samza.sh -t enrichment -k" - echo " - list: rb_samza.sh -l" - exit 1 -} - -f_execute=0 -f_kill=0 -f_list=0 -f_cont=0 -f_mem=0 -containers=1 -memory=1024 -f_version=0 -f_type="all" -f_show=0 - -while getopts "hevklt:c:m:s" name -do - case $name in - h) usage;; - e) f_execute=1;; - k) f_kill=1;; - l) f_list=1;; - v) f_version=1;; - t) f_type=$OPTARG;; - c) f_cont=1; containers=$OPTARG;; - m) f_mem=1; memory=$OPTARG;; - s) f_show=1;; - esac -done - -if [ $f_version -eq 1 ]; then - ls /var/rb-samza-bi/app/rb-samza-bi.tar.gz -l|sed 's/.*-> //'|awk '{print $1}' | sed 's/rb-samza-bi-//' | sed 's/-.*$//' -else - [ $f_cont -eq 1 ] && rb_set_samzacontainers.rb -c $containers - [ $f_mem -eq 1 ] && rb_set_samzacontainersmemory.rb -m $memory - [ $f_show -eq 1 ] && rb_set_samzacontainers.rb -s && rb_set_samzacontainersmemory.rb -s - if [ $f_kill -eq 1 ]; then - if [ "x$f_type" == "xenrichment" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep enrichment | awk '{print $1}')" - running="x$application"; - if [ $running != "x" ]; then - yarn application -kill $application &>/dev/null - echo "Killed application $application (enrichment)" - else - echo "The enrichment application was not running" - fi - fi - - if [ "x$f_type" == "xlocation" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep location | awk '{print $1}')" - running="x$application"; - if [ $running != "x" ]; then - yarn application -kill $application &>/dev/null - echo "Killed application $application (location)" - else - echo "The location application was not running" - fi - fi - - if [ "x$f_type" == "xindexing" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep indexing | awk '{print $1}')" - running="x$application"; - if [ $running != "x" ]; then - yarn application -kill $application &>/dev/null - echo "Killed application $application (indexing)" - else - echo "The indexing application was not running" - fi - fi - - if [ "x$f_type" == "xmalware" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep samza-malware | awk '{print $1}')" - running="x$application"; - if [ $running != "x" ]; then - yarn application -kill $application &>/dev/null - echo "Killed application $application (malware)" - else - echo "The malware application was not running" - fi - fi - - if [ "x$f_type" == "xpms" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep pms | awk '{print $1}')" - running="x$application"; - if [ $running != "x" ]; then - yarn application -kill $application &>/dev/null - echo "Killed application $application (pms)" - else - echo "The pms application was not running" - fi - fi - fi - - if [ $f_list -eq 1 ]; then - timeout 60 yarn application -list | egrep "(Samza|Application-Id)" - fi - - if [ $f_execute -eq 1 ]; then - if [ "x$f_type" == "xmalware" ]; then - - rm -rf /var/rb-samza-malware/bin/* - rm -rf /var/rb-samza-malware/lib/* - tar xfz /var/rb-samza-malware/app/rb-samza-malware.tar.gz -C /var/rb-samza-malware - - application="$(yarn application -list | grep samza-malware | awk '{print $1}')" - running="x$application"; - if [ $running == "x" ]; then - /var/rb-samza-malware/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-malware/config/malware.properties - else - echo "Application $application (malware) is already running." - fi - else - rm -rf /var/rb-samza-bi/bin/* - rm -rf /var/rb-samza-bi/lib/* - tar xfz /var/rb-samza-bi/app/rb-samza-bi.tar.gz -C /var/rb-samza-bi - - if [ "x$f_type" == "xenrichment" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep enrichment | awk '{print $1}')" - running="x$application"; - if [ $running == "x" ]; then - /var/rb-samza-bi/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-bi/config/enrichment.properties - else - echo "Application $application (enrichment) is already running." - fi - fi - - if [ "x$f_type" == "xlocation" ] || [ "x$f_type" == "xall" ]; then - rm -rf /var/rb-samza-location/bin/* - rm -rf /var/rb-samza-location/lib/* - tar xfz /var/rb-samza-location/app/rb-samza-location.tar.gz -C /var/rb-samza-location - - application="$(yarn application -list | grep location | awk '{print $1}')" - running="x$application"; - if [ $running == "x" ]; then - /var/rb-samza-location/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-location/config/location.properties - else - echo "Application $application (location) is already running." - fi - fi - - if [ "x$f_type" == "xindexing" ] || [ "x$f_type" == "xall" ]; then - application="$(yarn application -list | grep indexing | awk '{print $1}')" - running="x$application"; - if [ $running == "x" ]; then - /var/rb-samza-bi/bin/run-job.sh --config-factory=org.apache.samza.config.factories.PropertiesConfigFactory --config-path=file:/var/rb-samza-bi/config/indexing.properties - else - echo "Application $application (indexing) is already running." - fi - fi - fi - fi - -fi diff --git a/resources/bin/rb_update_darklist.sh b/resources/bin/rb_update_darklist.sh index c6b6baf0..9e8626e1 100644 --- a/resources/bin/rb_update_darklist.sh +++ b/resources/bin/rb_update_darklist.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash/samza +# Script to download the darklist.json file needed to enrichment of rb_flow and rb_event on logstash source /etc/profile.d/rvm.sh diff --git a/resources/bin/rb_upload_cookbooks.sh b/resources/bin/rb_upload_cookbooks.sh index f960b48c..bcb0cdcd 100755 --- a/resources/bin/rb_upload_cookbooks.sh +++ b/resources/bin/rb_upload_cookbooks.sh @@ -48,7 +48,7 @@ function upload_cookbook() { listCookbooks="rb-common rb-selinux cron zookeeper kafka druid http2k memcached chef-server consul - hadoop samza nginx geoip webui snmp mongodb rbmonitor rbscanner + nginx geoip webui snmp mongodb rbmonitor rbscanner f2k logstash pmacct minio postgresql rbdswatcher rbevents-counter rsyslog freeradius rbnmsp n2klocd rbale rbcep k2http rblogstatter rb-arubacentral rbcgroup rb-exporter rb-chrony rb-clamav rb-postfix keepalived snort barnyard2 rbaioutliers mem2incident rb-ai diff --git a/resources/chef/solo/webui-attributes.json b/resources/chef/solo/webui-attributes.json index 9b26918d..3ed9f974 100644 --- a/resources/chef/solo/webui-attributes.json +++ b/resources/chef/solo/webui-attributes.json @@ -1,6 +1,5 @@ { "run_list" : [ - "recipe[webui::configure_db]", "recipe[webui::configure_modules]" ] } diff --git a/resources/etc/mode-list.yml b/resources/etc/mode-list.yml index fc055e21..18608aa1 100644 --- a/resources/etc/mode-list.yml +++ b/resources/etc/mode-list.yml @@ -28,9 +28,6 @@ - name: 'http2k' description: 'Http2k node' -- - name: 'samza' - description: 'Samza node' - name: 'f2k' description: 'f2k flow collector node' diff --git a/resources/scripts/rb_backup_node.rb b/resources/scripts/rb_backup_node.rb index 4a910959..06d87153 100755 --- a/resources/scripts/rb_backup_node.rb +++ b/resources/scripts/rb_backup_node.rb @@ -328,7 +328,7 @@ def check_oper (cmd, mode, type, message) end # Stop chef-client - check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka hadoop_ stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ") + check_oper("rb_service stop chef druid awslogs rb-cloudwatch rb-monitor rb-workers rb-webui nprobe n2klocd memcached kafka stanchion riak zookeeper pgpool nginx freeradius postgresql keepalived", verbose, type, "Stoping all services ... ") # Restore the node check_oper("#{tar} #{path} -C /", verbose, type, "Restoring files ... ") `sed -i '/rb_aws_secondary_ip.sh/d' /etc/keepalived/keepalived.conf` @@ -430,7 +430,7 @@ def check_oper (cmd, mode, type, message) end # empty virtual ips on new environment - ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-hadoop_namenode", "ipvirtual-internal-hadoop_resourcemanager", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x| + ["ipvirtual-external-erchef", "ipvirtual-external-freeradius", "ipvirtual-external-kafka", "ipvirtual-external-n2klocd", "ipvirtual-external-n2kmobiled", "ipvirtual-external-nprobe", "ipvirtual-external-rb-reputation", "ipvirtual-external-rb-webui", "ipvirtual-external-riak", "ipvirtual-external-trap2kafka", "ipvirtual-internal-cep", "ipvirtual-internal-drill", "ipvirtual-internal-erchef", "ipvirtual-internal-kafka", "ipvirtual-internal-n2kmetricd", "ipvirtual-internal-oozie", "ipvirtual-internal-postgresql"].each do |x| db_temp = Chef::DataBagItem.load('rBglobal', x ) rescue db_temp = nil if !db_temp.nil? and !db_temp["ip"].nil? and db_temp["ip"]!="" db_temp["ip"] = "" diff --git a/resources/scripts/rb_get_topics.rb b/resources/scripts/rb_get_topics.rb index 5b2ab85c..42307bd6 100755 --- a/resources/scripts/rb_get_topics.rb +++ b/resources/scripts/rb_get_topics.rb @@ -98,7 +98,7 @@ def get_partitions(zk, topic) topics.each do |topic| next if topic=="rb_alarm" or topic=="__consumer_offsets" or topic=="app" - #next if topic.start_with?"__samza_" + index=0 if zk.exists?("#{topic_path(topic)}") partitions = zk.children(partitions_path(topic)).map{|k| k.to_i}.sort.uniq