diff --git a/csmtest/buckets/basic/allocation.sh b/csmtest/buckets/basic/allocation.sh index a58f5980d..8008b15b3 100755 --- a/csmtest/buckets/basic/allocation.sh +++ b/csmtest/buckets/basic/allocation.sh @@ -2,7 +2,7 @@ # # buckets/basic/node.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -126,7 +126,7 @@ check_return_flag_value $? 0 "Test Case 12: Checking allocation history table u # Test Case 13: csm_allocation_create with launch node input # Get utility node name -utility_node=`nodels utility | head -1` +utility_node=`nodels ${UTILITY} | head -1` ${CSM_PATH}/csm_allocation_create -j 1 -n ${SINGLE_COMPUTE} -l ${utility_node} > ${TEMP_LOG} 2>&1 check_return_flag_value $? 0 "Test Case 13: csm_allocation_create with launch node input" diff --git a/csmtest/buckets/basic/csm_ctrl_cmd.sh b/csmtest/buckets/basic/csm_ctrl_cmd.sh index d553b9423..3826d0b3b 100755 --- a/csmtest/buckets/basic/csm_ctrl_cmd.sh +++ b/csmtest/buckets/basic/csm_ctrl_cmd.sh @@ -2,7 +2,7 @@ # # buckets/basic/csm_ctrl_cmd.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -43,7 +43,8 @@ date >> ${LOG} echo "------------------------------------------------------------" >> ${LOG} # Get Target Compute Node -target_node=`nodels compute_B | head -1` +#target_node=`nodels ${COMPUTE_B} | head -1` +target_node=`echo ${COMPUTE_B} | cut -d ',' -f 1` # Start with Fresh CSM environment ${BASH_SOURCE%/*}/../../tools/dual_aggregator/shutdown_daemons.sh diff --git a/csmtest/buckets/basic/db_script.sh b/csmtest/buckets/basic/db_script.sh index dea9d2d21..0b7843356 100755 --- a/csmtest/buckets/basic/db_script.sh +++ b/csmtest/buckets/basic/db_script.sh @@ -2,7 +2,7 @@ # # buckets/basic/db_script.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -69,7 +69,7 @@ check_return_exit $? 2 "Test Case 5: Checking fvttestdb was deleted" # Cleanup rm -f ${TEMP_LOG} -rm -f /var/lib/pgsql/backups/fvttestdb* +rm -rf /var/lib/pgsql/backups/fvttestdb_* echo "------------------------------------------------------------" >> ${LOG} echo "CSM DB Scripts Bucket PASSED" >> ${LOG} diff --git a/csmtest/buckets/error_injection/allocation.sh b/csmtest/buckets/error_injection/allocation.sh index e98dba40e..f875ae968 100755 --- a/csmtest/buckets/error_injection/allocation.sh +++ b/csmtest/buckets/error_injection/allocation.sh @@ -2,7 +2,7 @@ # # buckets/error_injection/allocation.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -65,7 +65,7 @@ check_all_output "IN_SERVICE" check_return_exit $? 0 "Test Case 2: csm_allocation_create node is out of service - verify back IN_SERVICE" # Test Case 3: csm_allocation_create permission denied -su -c "${CSM_PATH}/csm_allocation_create -j 123 -n ${SINGLE_COMPUTE}" plundgr > ${TEMP_LOG} 2>&1 +su -c "${CSM_PATH}/csm_allocation_create -j 123 -n ${SINGLE_COMPUTE}" ${USER} > ${TEMP_LOG} 2>&1 check_return_exit $? 1 "Test Case 3: csm_allocation_create permission denied" # Test Case 4: csm_allocation_create prolog error 255 @@ -267,7 +267,7 @@ ${CSM_PATH}/csm_allocation_update_history -a ${allocation_id} -c "test comment" check_return_flag_value $? 25 "Test Case 39: csm_allocation_update_history before allocation deleted (i.e. added to history table)" # Test Case 40: csm_allocation_delete permission denied -su -c "${CSM_PATH}/csm_allocation_delete -a ${allocation_id}" plundgr > ${TEMP_LOG} 2>&1 +su -c "${CSM_PATH}/csm_allocation_delete -a ${allocation_id}" ${USER} > ${TEMP_LOG} 2>&1 check_return_flag_value $? 16 "Test Case 40: csm_allocation_delete permission denied" # Test Case 40: csm_allocation_delete success diff --git a/csmtest/buckets/error_injection/infrastructure.sh b/csmtest/buckets/error_injection/infrastructure.sh index 4a7b902e4..578b39c42 100755 --- a/csmtest/buckets/error_injection/infrastructure.sh +++ b/csmtest/buckets/error_injection/infrastructure.sh @@ -2,7 +2,7 @@ # # buckets/error_injection/infrastructure.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -31,7 +31,7 @@ LOG=${LOG_PATH}/buckets/error_injection/infrastructure.log CSM_PATH=/opt/ibm/csm/bin TEMP_LOG=${LOG_PATH}/buckets/error_injection/infrastructure_tmp.log FLAG_LOG=${LOG_PATH}/buckets/error_injection/infrastructure_flags.log -UTILIY=$(nodels utility) +UTILIY=$(nodels ${UTILITY}) if [ -f "${BASH_SOURCE%/*}/../../include/functions.sh" ] then @@ -63,9 +63,9 @@ check_return_flag_value $? 0 "Test Case 1: Validating Health Check output" # Test Case 2: Start utility daemon, health check, utility daemon filled in echo "Test Case 2: Start utility daemon, health check, utility daemon filled in" >> ${LOG} -xdsh utility "systemctl start csmd-utility" +xdsh ${UTILITY} "systemctl start csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 2: Utility is active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 2: Health Check" @@ -74,16 +74,16 @@ check_return_flag_value $? 0 "Test Case 2: Validating Health Check output" # Test Case 3: health check from utility, same info, local listed as utility echo "Test Case 3: health check from utility, same info, local listed as utility" >> ${LOG} -xdsh utility "${CSM_PATH}/csm_infrastructure_health_check -v" > ${TEMP_LOG} 2>&1 +xdsh ${UTILITY} "${CSM_PATH}/csm_infrastructure_health_check -v" > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 3: Health check from utility" check_all_output "MASTER: ${MASTER}" "Aggregators:0" "Utility Nodes:1" "Local_daemon: UTILITY: ${UTILITY}" check_return_flag_value $? 0 "Test Case 3: Validating Health Check output" # Test Case 4: Stop utility daemon, health check, utility daemon unresponsive echo "Test Case 4: Stop utility daemon, health check, utility daemon unresponsive" >> ${LOG} -xdsh utility "systemctl stop csmd-utility" +xdsh ${UTILITY} "systemctl stop csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 1 "Test Case 4: Utility is not active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 4: Health Check" @@ -92,9 +92,9 @@ check_return_flag_value $? 0 "Test Case 4: Validating Health Check output" # Test Case 5: Start utility daemon, health check, utility daemon responsive and bounced=1 echo "Test Case 5: Start utility daemon, health check, utility daemon responsive and bounced=1" >> ${LOG} -xdsh utility "systemctl start csmd-utility" +xdsh ${UTILITY} "systemctl start csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 5: Utility is active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 5: Health Check" @@ -107,7 +107,7 @@ systemctl stop csmd-master sleep 1 systemctl is-active csmd-master > /dev/null check_return_exit $? 3 "Test Case 6: Master is inactive" -xdsh utility "cat /var/log/ibm/csm/csm_utility.log" > ${TEMP_LOG} 2>&1 +xdsh ${UTILITY} "cat /var/log/ibm/csm/csm_utility.log" > ${TEMP_LOG} 2>&1 check_all_output "Transition from RUNMODE_CONFIGURED to: RUNMODE_DISCONNECTED" check_return_flag_value $? 0 "Test Case 6: Utility transition to RUNMODE_DISCONNECTED" @@ -117,7 +117,7 @@ systemctl start csmd-master sleep 1 systemctl is-active csmd-master > /dev/null check_return_exit $? 0 "Test Case 7: Master is active" -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 7: Utility is still active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 7: Health Check" @@ -227,9 +227,9 @@ check_return_flag_value $? 0 "Test Case 15: Validating Health Check output" # Test Case 16: Start utility daemon, health check, utility and aggregator daemon present echo "Test Case 16: Start utility daemon, health check, utility and aggregator daemon present" >> ${LOG} -xdsh utility "systemctl start csmd-utility" +xdsh ${UTILITY} "systemctl start csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 16: Utility is active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 16: Health Check" @@ -249,9 +249,9 @@ check_return_flag_value $? 0 "Test Case 17: Validating Health Check output" # Test Case 18: Stop utility daemon, health check, unresponsive aggregator and utility echo "Test Case 18: Stop utility daemon, health check, unresponsive aggregator and utility" >> ${LOG} -xdsh utility "systemctl stop csmd-utility" +xdsh ${UTILITY} "systemctl stop csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 1 "Test Case 18: Utility is not active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 18: Health Check" @@ -271,9 +271,9 @@ check_return_flag_value $? 0 "Test Case 19: Validating Health Check output" # Test Case 20: Start utility daemon, health check, responsive aggregator and utility and bounced=1 for both echo "Test Case 20: Start utility daemon, health check, responsive aggregator and utility and bounced=1 for both" >> ${LOG} -xdsh utility "systemctl start csmd-utility" +xdsh ${UTILITY} "systemctl start csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 20: Utility is active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 20: Health Check" @@ -289,7 +289,7 @@ check_return_exit $? 3 "Test Case 21: Master is not active" xdsh ${AGGREGATOR_A} "cat /var/log/ibm/csm/csm_aggregator.log" > ${TEMP_LOG} 2>&1 check_all_output "Transition from RUNMODE_CONFIGURED to: RUNMODE_DISCONNECTED" check_return_flag_value $? 0 "Test Case 21: Aggregator transitions to RUNMODE_DISCONNECTED" -xdsh utility "cat /var/log/ibm/csm/csm_utility.log" > ${TEMP_LOG} 2>&1 +xdsh ${UTILITY} "cat /var/log/ibm/csm/csm_utility.log" > ${TEMP_LOG} 2>&1 check_all_output "Transition from RUNMODE_CONFIGURED to: RUNMODE_DISCONNECTED" check_return_flag_value $? 0 "Test Case 21: Utility transitions to RUNMODE_DISCONNECTED" @@ -299,7 +299,7 @@ systemctl start csmd-master sleep 1 systemctl is-active csmd-master > /dev/null check_return_exit $? 0 "Test Case 22: Master is active" -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 22: Utility is active" xdsh ${AGGREGATOR_A} "systemctl is-active csmd-aggregator" > /dev/null check_return_exit $? 0 "Test Case 22: Aggregator is active" @@ -405,9 +405,9 @@ check_return_flag_value $? 0 "Test Case 29: Validating Health Check output" # Test Case 30: Start utility daemon, health check, utility and aggregator daemon present echo "Test Case 30: Start utility daemon, health check, utility and aggregator daemon present" >> ${LOG} -xdsh utility "systemctl start csmd-utility" +xdsh ${UTILITY} "systemctl start csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 0 "Test Case 30: Utility is active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 30: Health Check" @@ -441,9 +441,9 @@ check_return_flag_value $? 0 "Test Case 32: Validating csm_ras_event_query outpu # Test Case 33: Stop utility daemon, health check, utility disconnected echo "Test Case 33: Stop utility daemon, health check, utility disconnected" >> ${LOG} -xdsh utility "systemctl stop csmd-utility" +xdsh ${UTILITY} "systemctl stop csmd-utility" sleep 1 -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null check_return_exit $? 1 "Test Case 33: Utility is not active" ${CSM_PATH}/csm_infrastructure_health_check -v > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 33: Health Check" diff --git a/csmtest/buckets/error_injection/versioning.sh b/csmtest/buckets/error_injection/versioning.sh index e8faab973..cd7989870 100755 --- a/csmtest/buckets/error_injection/versioning.sh +++ b/csmtest/buckets/error_injection/versioning.sh @@ -2,7 +2,7 @@ # # buckets/error_injection/versioning.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -70,7 +70,7 @@ systemctl start csmd-master > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Starting Master" xdsh ${AGGREGATOR_A} "systemctl start csmd-aggregator" > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Starting Aggregator" -xdsh utility "systemctl start csmd-utility" > ${TEMP_LOG} 2>&1 +xdsh ${UTILITY} "systemctl start csmd-utility" > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Starting Utility" xdsh ${SINGLE_COMPUTE} "systemctl start csmd-compute" > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Starting Compute" @@ -79,7 +79,7 @@ check_return_exit $? 0 "Starting Compute" systemctl status csmd-master > ${TEMP_LOG} 2>&1 check_return_exit $? 0 "Test Case 1: Check Master status" -sleep 5 +sleep 10 # Test Case 2: Check Compute is not active xdsh ${SINGLE_COMPUTE} "systemctl is-active csmd-compute" > ${TEMP_LOG} 2>&1 check_return_flag_value $? 1 "Test Case 2: Check Compute is not active" diff --git a/csmtest/config.template b/csmtest/config.template index c603d688e..1b222edf3 100644 --- a/csmtest/config.template +++ b/csmtest/config.template @@ -13,6 +13,7 @@ export RPM_DIR= export BIG_DATA_DIR= export SQL_DIR= export SSL_KEY= +export CAST_ELASTIC= # Node Types export XCATMN= @@ -22,8 +23,15 @@ export AGGREGATOR_B= export COMPUTE_NODES= export SINGLE_COMPUTE= export UFM_ADDR= +export XCATMN_IP= export LOGSTASH= +# Node Types +export SERVICE= +export COMPUTE_B= +export COMPUTE_A= +export UTILITY= + # Users export USER= export SECOND_USER= diff --git a/csmtest/include/functions.sh b/csmtest/include/functions.sh index ffc30efb7..37f3ec2ce 100755 --- a/csmtest/include/functions.sh +++ b/csmtest/include/functions.sh @@ -13,6 +13,15 @@ # #================================================================================ +# Try to source the configuration file to get global configuration variables +if [ -f "${BASH_SOURCE%/*}/../csm_test.cfg" ] +then + . "${BASH_SOURCE%/*}/../csm_test.cfg" +else + echo "Could not find csm_test.cfg file expected at "${BASH_SOURCE%/*}/../csm_test.cfg", exitting." + exit 1 +fi + # ---------------------------------------------------------------- # LogMsg Date/Time Function # Functionality: This helps track the date/timing of each bucket @@ -48,8 +57,8 @@ check_all_output () { shutdown_daemons () { systemctl stop csmd-master xdsh ${AGGREGATOR_A} "systemctl stop csmd-aggregator" - xdsh utility "systemctl stop csmd-utility" - xdsh csm_comp "systemctl stop csmd-compute" + xdsh ${UTILITY} "systemctl stop csmd-utility" + xdsh ${COMPUTE_NODES} "systemctl stop csmd-compute" echo "y" | /opt/ibm/csm/db/csm_db_script.sh -d csmdb /opt/ibm/csm/db/csm_db_script.sh -n csmdb } diff --git a/csmtest/include/hcdiag/csm_fvt_diag_test_properties_setup_p8_p9.sh b/csmtest/include/hcdiag/csm_fvt_diag_test_properties_setup_p8_p9.sh index d27b05e12..826b93614 100755 --- a/csmtest/include/hcdiag/csm_fvt_diag_test_properties_setup_p8_p9.sh +++ b/csmtest/include/hcdiag/csm_fvt_diag_test_properties_setup_p8_p9.sh @@ -13,6 +13,14 @@ # #================================================================================ +if [ -f "${BASH_SOURCE%/*}/../../csm_test.cfg" ] +then + . "${BASH_SOURCE%/*}/../../csm_test.cfg" +else + echo "Could not find csm_test.cfg file expected at "${BASH_SOURCE%/*}/../csm_test.cfg", exitting." + exit 1 +fi + #--------------------------------------- # Output formatter #--------------------------------------- @@ -212,7 +220,7 @@ fi # Check to ensure the return count is not -gt 1. # Find the pattern in the test.properties file to replace. #------------------------------------------------------------- -smt_set=`xdsh $COMPUTE_NODES "/usr/sbin/ppc64_cpu --smt | cut -d '=' -f2" | xcoll | grep -v csm_comp | grep -v ==================================== | grep -v "No route to host" | grep -v compute | awk 'NR>1 {print last} {last=$0}'` +smt_set=`xdsh $COMPUTE_NODES "/usr/sbin/ppc64_cpu --smt | cut -d '=' -f2" | xcoll | grep -v $COMPUTE_NODES | grep -v ==================================== | grep -v "No route to host" | grep -v compute | awk 'NR>1 {print last} {last=$0}'` wc_smt_set=$(echo -n $smt_set | wc -w) smt_tp_file=`grep -A 3 "\[tests.chk-smt\\b" $ORIG_FILE | awk 'FNR == 4 {print}' | awk '{ print $3}'` diff --git a/csmtest/setup/csm_dual_aggregator_setup.sh b/csmtest/setup/csm_dual_aggregator_setup.sh index 686d19c23..b344261d2 100755 --- a/csmtest/setup/csm_dual_aggregator_setup.sh +++ b/csmtest/setup/csm_dual_aggregator_setup.sh @@ -2,7 +2,7 @@ # # setup/csm_dual_aggregator_setup.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -35,15 +35,15 @@ rm -f ${INSTALL_DIR}/ibm-* # Clean local rpms directory on Aggregator, Utility, Comp nodes xdsh ${AGGREGATOR_A} "rm -rf /root/rpms/*.rpm" xdsh ${AGGREGATOR_B} "rm -rf /root/rpms/*.rpm" -xdsh csm_comp,utility "rm -rf /root/rpms" +xdsh ${COMPUTE_NODES},${UTILITY} "rm -rf /root/rpms" # Grab Daily Build cp ${RPM_DIR}/ibm-csm-* ${INSTALL_DIR} cp ${RPM_DIR}/ibm-flightlog-1.* ${INSTALL_DIR} # Stop old CSM deamons on Utility, Computes & Aggregator -xdsh csm_comp "systemctl stop csmd-compute" -xdsh utility "systemctl stop csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl stop csmd-compute" +xdsh ${UTILITY} "systemctl stop csmd-utility" xdsh ${AGGREGATOR_A} "systemctl stop csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl stop csmd-aggregator" @@ -55,12 +55,12 @@ systemctl stop csmd-master rpm -e ibm-csm-core-* ibm-csm-api-* ibm-csm-db-* ibm-csm-hcdiag-* ibm-csm-restd-* ibm-flightlog-* # Remove old CSM RPMs on Utility, Computes & Aggregator -xdsh csm_comp,utility "rpm -e ibm-csm-core-* ibm-csm-api-* ibm-csm-hcdiag-* ibm-flightlog-*" +xdsh ${COMPUTE_NODES},${UTILITY} "rpm -e ibm-csm-core-* ibm-csm-api-* ibm-csm-hcdiag-* ibm-flightlog-*" xdsh ${AGGREGATOR_A} "rpm -e ibm-csm-core-* ibm-csm-api-* ibm-csm-hcdiag-* ibm-flightlog-*" xdsh ${AGGREGATOR_B} "rpm -e ibm-csm-core-* ibm-csm-api-* ibm-csm-hcdiag-* ibm-flightlog-*" # File clean up -xdsh csm_comp,utility "rm -rf /etc/ibm /var/log/ibm" +xdsh ${COMPUTE_NODES},${UTILITY} "rm -rf /etc/ibm /var/log/ibm" rm -rf /etc/ibm /var/log/ibm xdsh ${AGGREGATOR_A} "rm -rf /etc/ibm /var/log/ibm" xdsh ${AGGREGATOR_B} "rm -rf /etc/ibm /var/log/ibm" @@ -88,26 +88,26 @@ xdsh ${AGGREGATOR_B} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" xdsh ${AGGREGATOR_B} "rpm -ivh /root/rpms/ibm-csm-api-*" # Install new CSM RPMs - Utility node (login + launch) -xdcp utility -R ${INSTALL_DIR} /root/ -xdsh utility "rpm -ivh /root/rpms/ibm-flightlog-1.*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-core-*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-api-*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" +xdcp ${UTILITY} -R ${INSTALL_DIR} /root/ +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-flightlog-1.*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-core-*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-api-*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" # Install new CSM RPMs - Compute node -xdcp csm_comp -R ${INSTALL_DIR} /root -xdsh csm_comp "rpm -ivh /root/rpms/ibm-flightlog-1.*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-core-*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-api-*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" +xdcp ${COMPUTE_NODES} -R ${INSTALL_DIR} /root +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-flightlog-1.*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-core-*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-api-*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" # CSM DB cleanup echo "y" | /opt/ibm/csm/db/csm_db_script.sh -d csmdb @@ -141,31 +141,31 @@ sed -i -- "s/__AGGREGATOR_B__/${AGGREGATOR_A}/g" csm_compute_B.cfg # Configuration file distribution cd ${FVT_PATH}/setup/ -xdsh csm_comp,utility "mkdir -p /etc/ibm/csm" +xdsh ${COMPUTE_NODES},${UTILITY} "mkdir -p /etc/ibm/csm" xdsh ${AGGREGATOR_A} "mkdir -p /etc/ibm/csm" xdsh ${AGGREGATOR_B} "mkdir -p /etc/ibm/csm" -xdcp compute_A /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg -xdcp compute_B /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg -xdcp utility /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg +xdcp ${COMPUTE_A} /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg +xdcp ${COMPUTE_B} /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg +xdcp ${UTILITY} /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_aggregator_A.cfg /etc/ibm/csm/csm_aggregator.cfg xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_aggregator_B.cfg /etc/ibm/csm/csm_aggregator.cfg -xdcp csm_comp,utility /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl +xdcp ${COMPUTE_NODES},${UTILITY} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg -xdcp csm_comp,utility /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg +xdcp ${COMPUTE_NODES},${UTILITY} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg # 4.2.3 Prolog/Epilog Scripts Compute -xdcp csm_comp /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs +xdcp ${COMPUTE_NODES} /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs # Start Daemons systemctl start csmd-master xdsh ${AGGREGATOR_A} "systemctl start csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl start csmd-aggregator" -xdsh utility "systemctl start csmd-utility" -xdsh csm_comp "systemctl start csmd-compute" +xdsh ${UTILITY} "systemctl start csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl start csmd-compute" # Daemon Reload systemctl daemon-reload -xdsh csm_comp,utility "systemctl daemon-reload" +xdsh ${COMPUTE_NODES},${UTILITY} "systemctl daemon-reload" diff --git a/csmtest/setup/csm_install.sh b/csmtest/setup/csm_install.sh index 652e4d761..2d544ec8e 100755 --- a/csmtest/setup/csm_install.sh +++ b/csmtest/setup/csm_install.sh @@ -43,13 +43,13 @@ compute="" compute="${COMPUTE_NODES}" utility="" -utility="utility" +utility="${UTILITY}" service="" service="${AGGREGATOR_A},${AGGREGATOR_B}" master="" -master="localhost" +master="${MASTER}" all="" @@ -73,10 +73,10 @@ fi master_node=`hostname` # Get list of compute nodes -compute_node_list=`nodels csm_comp` +#compute_node_list=`nodels ${COMPUTE_NODES}` # Get list of utility nodes -utility_node_list=`nodels utility` +utility_node_list="$utility" #---------------------------------------------------------------------- # RPM Replace and Download Section @@ -133,7 +133,7 @@ do done # Replace Old RPMs in /root/rpms on Compute nodes with RPMs from INSTALL_DIR -for node in ${compute_node_list} +for node in $( echo $COMPUTE_NODES | sed "s/,/ /g") do ssh ${node} "ls /root/rpms/ibm-csm-core-*" > /dev/null 2>&1 if [ $? -eq 0 ] @@ -188,7 +188,7 @@ do done # Install RPMs on Compute Nodes -for node in ${compute_node_list} +for node in $( echo $COMPUTE_NODES | sed "s/,/ /g") do xdsh ${node} "rpm -ivh /root/rpms/ibm-flightlog-1.*" xdsh ${node} "rpm -ivh /root/rpms/ibm-csm-core-*" @@ -251,24 +251,24 @@ if [ ! -z $LOGSTASH ] fi # Configuration file distribution -xdsh csm_comp,utility "mkdir -p /etc/ibm/csm" +xdsh ${COMPUTE_NODES},${UTILITY} "mkdir -p /etc/ibm/csm" xdsh ${AGGREGATOR_A} "mkdir -p /etc/ibm/csm" xdsh ${AGGREGATOR_B} "mkdir -p /etc/ibm/csm" -xdcp utility /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg +xdcp ${UTILITY} /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_aggregator_A.cfg /etc/ibm/csm/csm_aggregator.cfg xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_aggregator_B.cfg /etc/ibm/csm/csm_aggregator.cfg -xdcp csm_comp,utility /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl +xdcp ${COMPUTE_NODES},${UTILITY} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg -xdcp csm_comp,utility /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg +xdcp ${COMPUTE_NODES},${UTILITY} /etc/ibm/csm/csm_api.cfg /etc/ibm/csm/csm_api.cfg if [ ${AGGREGATOR_A} == ${AGGREGATOR_B} ] then - xdcp csm_comp /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg + xdcp ${COMPUTE_NODES} /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg else - xdcp compute_A /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg - xdcp compute_B /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg + xdcp ${COMPUTE_A} /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg + xdcp ${COMPUTE_B} /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg fi if [ ! -z $SSL_KEY ] then @@ -277,15 +277,15 @@ if [ ! -z $SSL_KEY ] fi # 4.2.3 Prolog/Epilog Scripts Compute -xdcp csm_comp -p /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs -xdsh csm_comp,utility "/usr/bin/cp -p /opt/ibm/csm/share/recovery/soft_failure_recovery /opt/ibm/csm/recovery/soft_failure_recovery" +xdcp ${COMPUTE_NODES} -p /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs +xdsh ${COMPUTE_NODES},${UTILITY} "/usr/bin/cp -p /opt/ibm/csm/share/recovery/soft_failure_recovery /opt/ibm/csm/recovery/soft_failure_recovery" # Set up the pam modules on the CSM compute nodes -xdsh csm_comp "if [ -e /usr/lib64/security/libcsmpam.so ] ; then sed -i '/libcsmpam.so/s/^#*//g' /etc/pam.d/sshd ; fi" +xdsh ${COMPUTE_NODES} "if [ -e /usr/lib64/security/libcsmpam.so ] ; then sed -i '/libcsmpam.so/s/^#*//g' /etc/pam.d/sshd ; fi" # Daemon Reload systemctl daemon-reload -xdsh csm_comp,utility "systemctl daemon-reload" +xdsh ${COMPUTE_NODES},${UTILITY} "systemctl daemon-reload" # Start Nvidia daemons if [ -n "${all}" ] ; then @@ -297,5 +297,5 @@ fi systemctl start csmd-master xdsh ${AGGREGATOR_A} "systemctl start csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl start csmd-aggregator" -xdsh utility "systemctl start csmd-utility" -xdsh csm_comp "systemctl start csmd-compute" +xdsh ${UTILITY} "systemctl start csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl start csmd-compute" diff --git a/csmtest/setup/csm_setup.sh b/csmtest/setup/csm_setup.sh index 2737264a6..fd9ce37c1 100755 --- a/csmtest/setup/csm_setup.sh +++ b/csmtest/setup/csm_setup.sh @@ -38,18 +38,21 @@ else exit 1 fi +# Output formatter +line1_out=$(printf "%0.s-" {1..90}) + # Get hostname of master node compute="" compute="${COMPUTE_NODES}" utility="" -utility="utility" +utility="${UTILITY}" service="" service="${AGGREGATOR_A},${AGGREGATOR_B}" master="" -master="localhost" +master="${MASTER}" all="" @@ -73,50 +76,89 @@ fi master_node=`hostname` # Get list of compute nodes -compute_node_list=`nodels csm_comp` +compute_node_list="$compute" # Get list of utility nodes -utility_node_list=`nodels utility` +utility_node_list="$utility" # Clean local rpms directory rm -f ${INSTALL_DIR}/ibm-* # Clean local rpms directory on Aggregator, Utility, Comp nodes xdsh ${AGGREGATOR_A} "rm -rf /root/rpms/*.rpm" -xdsh csm_comp,utility "rm -rf /root/rpms" +xdsh ${COMPUTE_NODES},${UTILITY} "rm -rf /root/rpms" # Grab Daily Build cp ${RPM_DIR}/ibm-csm-* ${INSTALL_DIR} cp ${RPM_DIR}/ibm-flightlog-1.* ${INSTALL_DIR} -# Stop old CSM deamons on Utility, Computes & Aggregator -xdsh csm_comp "systemctl stop csmd-compute" -xdsh utility "systemctl stop csmd-utility" -xdsh ${AGGREGATOR_A} "systemctl stop csmd-aggregator" +# Stop CSM daemons on Compute +for node in $( echo $COMPUTE_NODES | sed "s/,/ /g") +do + ssh ${node} "systemctl is-active csmd-compute" > /dev/null + if [ $? -eq 0 ] + then + ssh ${node} "systemctl stop csmd-compute" > /dev/null + fi +done + +# Stop CSM daemons on Utility +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null +if [ $? -eq 0 ] + then + xdsh ${UTILITY} "systemctl stop csmd-utility" > /dev/null +fi + +# Stop CSM daemons on Aggregator +xdsh ${AGGREGATOR_A} "systemctl is-active csmd-aggregator" > /dev/null +if [ $? -eq 0 ] + then + xdsh ${AGGREGATOR_A} "systemctl stop csmd-aggregator" > /dev/null +fi -# Stop old CSM deamons on Master -systemctl stop csmd-aggregator -systemctl stop csmd-master +# Stop CSM daemons on Master +systemctl is-active csmd-master > /dev/null +if [ $? -eq 0 ] + then + systemctl stop csmd-master > /dev/null +fi +systemctl is-active csmd-aggregator > /dev/null +if [ $? -eq 0 ] + then + systemctl stop csmd-aggregator > /dev/null +fi # Remove old CSM RPMs on Master -RPMS="ibm-csm-bds ibm-csm-unittest ibm-csm-db ibm-csm-hcdiag ibm-csm-restd ibm-csm-api ibm-csm-core ibm-flightlog ibm-csm-bds-logstash" -echo "------------------------------------------------------------------------------------------------------------------------" +RPMS="ibm-csm-bds ibm-csm-bds-logstash ibm-csm-unittest ibm-csm-db ibm-csm-hcdiag ibm-csm-restd ibm-csm-api ibm-csm-core ibm-fshipmond ibm-fshipd ibm-fshipcld ibm-burstbuffer-tools ibm-burstbuffer-tests ibm-burstbuffer ibm-burstbuffer-mn ibm-burstbuffer-lsf ibm-csm-tools ibm-export_layout ibm-scripts ibm-transport-devel ibm-utilities-devel ibm-flightlog ibm-flightlog-devel" +echo "$line1_out" for RPM in $RPMS ; do if rpm --quiet -q "$RPM" ; then - rpm -e "$RPM" + rpm -e "$RPM" &> /dev/null printf "%-20s %-15s %-25s %-15s\n" "${MASTER}:" "Package:" "${RPM}" "Uninstalled" + else + printf "%-20s %-15s %-25s %-15s\n" "${MASTER}:" "Package:" "${RPM}" "Is Not installed" fi done -echo "------------------------------------------------------------------------------------------------------------------------" # Remove all existing rpms from utility, aggregator, and compute nodes if [ -n "${all}" ] ; then - xdsh ${all} "for RPM in $RPMS ; do if rpm --quiet -q \$RPM ; then rpm -e \$RPM ; printf \"%-20s %-15s %-25s %-15s\n\" \"Group_Names_Above:\" \"Package:\" \"\${RPM}\" \"Uninstalled\"; fi done" | xcoll +echo "$line1_out" +xdsh ${all} \ +"for RPM in $RPMS ; do \ + if rpm --quiet -q \$RPM ; then \ + rpm -e \$RPM &> /dev/null ; \ + if [ $? -eq 0 ]; then \ + printf \"%-24s %-25s %16s\n\" \" Package:\" \$RPM \"Uninstalled\" ; \ + fi \ + else + printf \"%-24s %-25s %15s\n\" \" Package:\" \$RPM \"Is Not installed\" ; \ + fi \ +done" # File clean up xdsh ${all} "rm -rf /etc/ibm /var/log/ibm" fi -echo "------------------------------------------------------------------------------------------------------------------------" +echo "$line1_out" # Install new CSM RPMs - Management node rpm -ivh ${INSTALL_DIR}/ibm-flightlog-1.* @@ -136,26 +178,26 @@ xdsh ${AGGREGATOR_A} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" xdsh ${AGGREGATOR_A} "rpm -ivh /root/rpms/ibm-csm-api-*" # Install new CSM RPMs - Utility node (login + launch) -xdcp utility -R ${INSTALL_DIR} /root/ -xdsh utility "rpm -ivh /root/rpms/ibm-flightlog-1.*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-core-*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-api-*" -xdsh utility "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" -#xdsh utility "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" +xdcp ${UTILITY} -R ${INSTALL_DIR} /root/ +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-flightlog-1.*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-core-*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-api-*" +xdsh ${UTILITY} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" +#xdsh ${UTILITY} "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" # Install new CSM RPMs - Compute node -xdcp csm_comp -R ${INSTALL_DIR} /root -xdsh csm_comp "rpm -ivh /root/rpms/ibm-flightlog-1.*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-core-*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-api-*" -xdsh csm_comp "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" -#xdsh csm_comp "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" +xdcp ${COMPUTE_NODES} -R ${INSTALL_DIR} /root +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-flightlog-1.*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-core-*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-api-*" +xdsh ${COMPUTE_NODES} "rpm -ivh /root/rpms/ibm-csm-hcdiag-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-flightlog-0.2.0-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-core-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-api-*" +#xdsh ${COMPUTE_NODES} "rpm -ivh ${INSTALL_DIR}/ibm-csm-hcdiag-*" # CSM DB cleanup echo "y" | /opt/ibm/csm/db/csm_db_script.sh -d csmdb @@ -176,20 +218,20 @@ sed -i -- "s/__UFM_REST_ADDRESS__/${UFM_ADDR}/g" * #sed -i -- "s/__AGGREGATOR_B__/${AGGREGATOR_B}/g" * #sed -i -- "s/__CSMRESTD_IP__/${localhost}/g" * cd ${FVT_PATH}/setup/ -xdsh csm_comp,utility "mkdir -p /etc/ibm/csm/" -xdcp csm_comp /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg -xdcp utility /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg +xdsh ${COMPUTE_NODES},${UTILITY} "mkdir -p /etc/ibm/csm/" +xdcp ${COMPUTE_NODES} /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg +xdcp ${UTILITY} /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_aggregator.cfg /etc/ibm/csm/csm_aggregator.cfg -xdcp csm_comp,utility /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl +xdcp ${COMPUTE_NODES},${UTILITY} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_api.acl /etc/ibm/csm/csm_api.acl # 4.2.3 Prolog/Epilog Scripts Compute -xdcp csm_comp -p /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs -xdsh csm_comp,utility "/usr/bin/cp -p /opt/ibm/csm/share/recovery/soft_failure_recovery /opt/ibm/csm/recovery/soft_failure_recovery" +xdcp ${COMPUTE_NODES} -p /opt/ibm/csm/share/prologs/* /opt/ibm/csm/prologs +xdsh ${COMPUTE_NODES},${UTILITY} "/usr/bin/cp -p /opt/ibm/csm/share/recovery/soft_failure_recovery /opt/ibm/csm/recovery/soft_failure_recovery" # Daemon Reload systemctl daemon-reload -xdsh csm_comp,utility "systemctl daemon-reload" +xdsh ${COMPUTE_NODES},${UTILITY} "systemctl daemon-reload" # Start Nvidia daemons if [ -n "${all}" ] ; then @@ -200,5 +242,5 @@ fi # Start Daemons systemctl start csmd-master xdsh ${AGGREGATOR_A} "systemctl start csmd-aggregator" -xdsh utility "systemctl start csmd-utility" -xdsh csm_comp "systemctl start csmd-compute" +xdsh ${UTILITY} "systemctl start csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl start csmd-compute" diff --git a/csmtest/setup/csm_uninstall.sh b/csmtest/setup/csm_uninstall.sh index 74c948caf..63291e3e8 100755 --- a/csmtest/setup/csm_uninstall.sh +++ b/csmtest/setup/csm_uninstall.sh @@ -16,7 +16,7 @@ # Script to uninstall CSM on test environment # DEBUG: -# set -x +#set -x # Try to source the configuration file to get global configuration variables if [ -f "${BASH_SOURCE%/*}/../csm_test.cfg" ] @@ -42,23 +42,22 @@ fi line1_out=$(printf "%0.s-" {1..90}) # Get list of compute nodes -compute_node_list=`nodels csm_comp` +#compute_node_list=`nodels ${COMPUTE_NODES}` # Get list of utility nodes -utility_node_list=`nodels utility` +#utility_node_list=`nodels ${UTILITY}` # Get hostname of master node - compute="" compute="${COMPUTE_NODES}" utility="" -utility="${utility_node_list}" +utility="${UTILITY}" service="" service="${AGGREGATOR_A},${AGGREGATOR_B}" master="" -master="localhost" +master="${MASTER}" all="" @@ -80,7 +79,7 @@ if [ -n "${all}" ] ; then fi # Comment out any entries for libcsmpam.so in /etc/pam.d/sshd -xdsh all "sed -i '/libcsmpam.so/s/^#*/#/g' /etc/pam.d/sshd" +xdsh ${all} "sed -i '/libcsmpam.so/s/^#*/#/g' /etc/pam.d/sshd" sleep 1 # Stop CSM daemons on Master @@ -96,7 +95,7 @@ if [ $? -eq 0 ] fi # Stop CSM daemons on Compute -for node in ${compute_node_list} +for node in $( echo $COMPUTE_NODES | sed "s/,/ /g") do ssh ${node} "systemctl is-active csmd-compute" > /dev/null if [ $? -eq 0 ] @@ -106,10 +105,10 @@ do done # Stop CSM daemons on Utility -xdsh utility "systemctl is-active csmd-utility" > /dev/null +xdsh ${UTILITY} "systemctl is-active csmd-utility" > /dev/null if [ $? -eq 0 ] then - xdsh utility "systemctl stop csmd-utility" > /dev/null + xdsh ${UTILITY} "systemctl stop csmd-utility" > /dev/null fi # Stop CSM daemons on Aggregator @@ -151,8 +150,7 @@ xdsh ${all} \ else printf \"%-24s %-25s %15s\n\" \" Package:\" \$RPM \"Is Not installed\" ; \ fi \ -done \ -" +done" # Files cleanup xdsh ${all} "rm -rf /etc/ibm /var/log/ibm" @@ -163,7 +161,7 @@ if [ -n "${master}" ] ; then fi # RPM directory clean up -xdsh csm_comp,utility "rm -rf /root/rpms" +xdsh ${COMPUTE_NODES},${UTILITY} "rm -rf /root/rpms" if [ `hostname` != ${AGGREGATOR_A} ] then xdsh ${AGGREGATOR_A} "rm -rf /root/rpms" diff --git a/csmtest/tools/ci_fvt.sh b/csmtest/tools/ci_fvt.sh index 40b84aa36..1ef38671e 100755 --- a/csmtest/tools/ci_fvt.sh +++ b/csmtest/tools/ci_fvt.sh @@ -2,7 +2,7 @@ # # tools/complete_fvt.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -41,8 +41,8 @@ segfault_scan () { xdcp ${MASTER} -P /var/log/ibm/csm/csm_master.log ${LOG_PATH}/performance/ xdcp ${AGGREGATOR_A} -P /var/log/ibm/csm/csm_aggregator.log ${LOG_PATH}/performance/ xdcp ${AGGREGATOR_B} -P /var/log/ibm/csm/csm_aggregator.log ${LOG_PATH}/performance/ - xdcp utility -P /var/log/ibm/csm/csm_utility.log ${LOG_PATH}/performance/ - xdcp csm_comp -P /var/log/ibm/csm/csm_compute.log ${LOG_PATH}/performance/ + xdcp ${UTILITY} -P /var/log/ibm/csm/csm_utility.log ${LOG_PATH}/performance/ + xdcp ${COMPUTE_NODES} -P /var/log/ibm/csm/csm_compute.log ${LOG_PATH}/performance/ # Grep for segfault # Exit if found diff --git a/csmtest/tools/complete_fvt.sh b/csmtest/tools/complete_fvt.sh index 80075b9ca..5f8f6e3be 100755 --- a/csmtest/tools/complete_fvt.sh +++ b/csmtest/tools/complete_fvt.sh @@ -2,7 +2,7 @@ # # tools/complete_fvt.sh # -# © Copyright IBM Corporation 2015-2020. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -41,8 +41,8 @@ segfault_scan () { xdcp ${MASTER} -P /var/log/ibm/csm/csm_master.log ${LOG_PATH}/performance/ xdcp ${AGGREGATOR_A} -P /var/log/ibm/csm/csm_aggregator.log ${LOG_PATH}/performance/ xdcp ${AGGREGATOR_B} -P /var/log/ibm/csm/csm_aggregator.log ${LOG_PATH}/performance/ - xdcp utility -P /var/log/ibm/csm/csm_utility.log ${LOG_PATH}/performance/ - xdcp csm_comp -P /var/log/ibm/csm/csm_compute.log ${LOG_PATH}/performance/ + xdcp ${UTILITY} -P /var/log/ibm/csm/csm_utility.log ${LOG_PATH}/performance/ + xdcp ${COMPUTE_NODES} -P /var/log/ibm/csm/csm_compute.log ${LOG_PATH}/performance/ # Grep for segfault # Exit if found @@ -120,7 +120,7 @@ ${FVT_PATH}/tools/dual_aggregator/shutdown_daemons.sh sleep 5 ${FVT_PATH}/setup/csm_setup.sh run_bucket "error_injection" "infrastructure" -run_bucket "error_injection" "versioning" +#run_bucket "error_injection" "versioning" ${FVT_PATH}/setup/csm_uninstall.sh ${FVT_PATH}/setup/csm_install.sh run_bucket "basic" "python_libraries" diff --git a/csmtest/tools/dual_aggregator/daemon_status.sh b/csmtest/tools/dual_aggregator/daemon_status.sh index 4806e6dd3..4f5da0749 100755 --- a/csmtest/tools/dual_aggregator/daemon_status.sh +++ b/csmtest/tools/dual_aggregator/daemon_status.sh @@ -2,7 +2,7 @@ # # tools/dual_aggregator/daemon_status.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -28,6 +28,6 @@ echo "AGGREGATORS" xdsh ${AGGREGATOR_A} "systemctl is-active csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl is-active csmd-aggregator" echo "UTILITY" -xdsh utility "systemctl is-active csmd-utility" +xdsh ${UTILITY} "systemctl is-active csmd-utility" echo "COMPUTE" -xdsh csm_comp "systemctl is-active csmd-compute" +xdsh ${COMPUTE_NODES} "systemctl is-active csmd-compute" diff --git a/csmtest/tools/dual_aggregator/shutdown_daemons.sh b/csmtest/tools/dual_aggregator/shutdown_daemons.sh index 75aa223aa..963728b64 100755 --- a/csmtest/tools/dual_aggregator/shutdown_daemons.sh +++ b/csmtest/tools/dual_aggregator/shutdown_daemons.sh @@ -2,7 +2,7 @@ # # tools/dual_aggregator/daemon_shutdown.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -24,8 +24,8 @@ fi xdsh ${AGGREGATOR_A} "systemctl stop csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl stop csmd-aggregator" -xdsh utility "systemctl stop csmd-utility" -xdsh csm_comp "systemctl stop csmd-compute" +xdsh ${UTILITY} "systemctl stop csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl stop csmd-compute" systemctl stop csmd-master echo "y" | /opt/ibm/csm/db/csm_db_script.sh -d csmdb /opt/ibm/csm/db/csm_db_script.sh -n csmdb diff --git a/csmtest/tools/dual_aggregator/start_daemons.sh b/csmtest/tools/dual_aggregator/start_daemons.sh index f4d1f0db6..16f9c2136 100755 --- a/csmtest/tools/dual_aggregator/start_daemons.sh +++ b/csmtest/tools/dual_aggregator/start_daemons.sh @@ -2,7 +2,7 @@ # # tools/dual_aggregator/start_daemons.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -25,5 +25,5 @@ fi systemctl start csmd-master xdsh ${AGGREGATOR_A} "systemctl start csmd-aggregator" xdsh ${AGGREGATOR_B} "systemctl start csmd-aggregator" -xdsh utility "systemctl start csmd-utility" -xdsh csm_comp "systemctl start csmd-compute" +xdsh ${UTILITY} "systemctl start csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl start csmd-compute" diff --git a/csmtest/tools/enable_ssl.sh b/csmtest/tools/enable_ssl.sh index 95f83a0cf..2e39e93f9 100755 --- a/csmtest/tools/enable_ssl.sh +++ b/csmtest/tools/enable_ssl.sh @@ -2,7 +2,7 @@ # # tools/enable_ssl.sh # -# © Copyright IBM Corporation 2015-2020. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -42,7 +42,7 @@ fi # Create cert directory on remote nodes xdsh ${MASTER} "mkdir -p /root/cert" -xdsh utility "mkdir -p /root/cert" +xdsh ${UTILITY} "mkdir -p /root/cert" xdsh ${AGGREGATOR_A} "mkdir -p /root/cert" xdsh ${AGGREGATOR_B} "mkdir -p /root/cert" xdsh ${COMPUTE_NODES} "mkdir -p /root/cert" @@ -50,8 +50,8 @@ xdsh ${COMPUTE_NODES} "mkdir -p /root/cert" # Distribute xCAT certificate authority, credential files xdcp ${MASTER} /root/cert/ca.pem /root/cert/ xdcp ${MASTER} /root/cert/server-cred.pem /root/cert/ -xdcp utility /root/cert/ca.pem /root/cert/ -xdcp utility /root/cert/server-cred.pem /root/cert/ +xdcp ${UTILITY} /root/cert/ca.pem /root/cert/ +xdcp ${UTILITY} /root/cert/server-cred.pem /root/cert/ xdcp ${AGGREGATOR_A} /root/cert/ca.pem /root/cert/ xdcp ${AGGREGATOR_A} /root/cert/server-cred.pem /root/cert/ xdcp ${AGGREGATOR_B} /root/cert/ca.pem /root/cert/ @@ -80,13 +80,13 @@ sed -i -- "/cred_pem/c\ \"cred_pem\": \"/root/cert/server-cred.pe # Distribute CSM config files xdcp ${MASTER} /etc/ibm/csm/csm_master.cfg /etc/ibm/csm/csm_master.cfg -xdcp utility /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg +xdcp ${UTILITY} /etc/ibm/csm/csm_utility.cfg /etc/ibm/csm/csm_utility.cfg xdcp ${AGGREGATOR_A} /etc/ibm/csm/csm_aggregator_A.cfg /etc/ibm/csm/csm_aggregator.cfg xdcp ${AGGREGATOR_B} /etc/ibm/csm/csm_aggregator_B.cfg /etc/ibm/csm/csm_aggregator.cfg -xdcp compute_A /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg -xdcp compute_B /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg +xdcp ${COMPUTE_A} /etc/ibm/csm/csm_compute_A.cfg /etc/ibm/csm/csm_compute.cfg +xdcp ${COMPUTE_B} /etc/ibm/csm/csm_compute_B.cfg /etc/ibm/csm/csm_compute.cfg # If csm_compute_A/B.cfg does not exist, distribute in single aggregator config if [ ! -f /etc/ibm/csm/csm_compute_A.cfg ] then - xdcp csm_comp /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg + xdcp ${COMPUTE_NODES} /etc/ibm/csm/csm_compute.cfg /etc/ibm/csm/csm_compute.cfg fi diff --git a/csmtest/tools/single_aggregator/daemon_status.sh b/csmtest/tools/single_aggregator/daemon_status.sh index 9e6d0ea97..f8209719e 100755 --- a/csmtest/tools/single_aggregator/daemon_status.sh +++ b/csmtest/tools/single_aggregator/daemon_status.sh @@ -2,7 +2,7 @@ # # tools/single_aggregator/daemon_status.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -13,11 +13,19 @@ # #================================================================================ +if [ -f "${BASH_SOURCE%/*}/../../csm_test.cfg" ] +then + . "${BASH_SOURCE%/*}/../../csm_test.cfg" +else + echo "Could not find csm_test.cfg file expected at "${BASH_SOURCE%/*}/../../csm_test.cfg", exitting." + exit 1 +fi + echo "MASTER" systemctl is-active csmd-master echo "AGGREGATOR" systemctl is-active csmd-aggregator echo "UTILITY" -xdsh utility "systemctl is-active csmd-utility" +xdsh ${UTILITY} "systemctl is-active csmd-utility" echo "COMPUTE" -xdsh csm_comp "systemctl is-active csmd-compute" +xdsh ${COMPUTE_NODES} "systemctl is-active csmd-compute" diff --git a/csmtest/tools/single_aggregator/shutdown_daemons.sh b/csmtest/tools/single_aggregator/shutdown_daemons.sh index afaf4a733..077453999 100755 --- a/csmtest/tools/single_aggregator/shutdown_daemons.sh +++ b/csmtest/tools/single_aggregator/shutdown_daemons.sh @@ -2,7 +2,7 @@ # # tools/single_aggregator/shutdown_daemons.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -13,9 +13,17 @@ # #================================================================================ +if [ -f "${BASH_SOURCE%/*}/../../csm_test.cfg" ] +then + . "${BASH_SOURCE%/*}/../../csm_test.cfg" +else + echo "Could not find csm_test.cfg file expected at "${BASH_SOURCE%/*}/../../csm_test.cfg", exitting." + exit 1 +fi + systemctl stop csmd-aggregator -xdsh utility "systemctl stop csmd-utility" -xdsh csm_comp "systemctl stop csmd-compute" +xdsh ${UTILITY} "systemctl stop csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl stop csmd-compute" systemctl stop csmd-master echo "y" | /opt/ibm/csm/db/csm_db_script.sh -d csmdb /opt/ibm/csm/db/csm_db_script.sh -n csmdb diff --git a/csmtest/tools/single_aggregator/start_daemons.sh b/csmtest/tools/single_aggregator/start_daemons.sh index 2187fea74..a295c0f7c 100755 --- a/csmtest/tools/single_aggregator/start_daemons.sh +++ b/csmtest/tools/single_aggregator/start_daemons.sh @@ -2,7 +2,7 @@ # # tools/single_aggregator/start_daemons.sh # -# © Copyright IBM Corporation 2015-2018. All Rights Reserved +# © Copyright IBM Corporation 2015-2021. All Rights Reserved # # This program is licensed under the terms of the Eclipse Public License # v1.0 as published by the Eclipse Foundation and available at @@ -13,7 +13,15 @@ # #================================================================================ +if [ -f "${BASH_SOURCE%/*}/../../csm_test.cfg" ] +then + . "${BASH_SOURCE%/*}/../../csm_test.cfg" +else + echo "Could not find csm_test.cfg file expected at "${BASH_SOURCE%/*}/../../csm_test.cfg", exitting." + exit 1 +fi + systemctl start csmd-master systemctl start csmd-aggregator -xdsh utility "systemctl start csmd-utility" -xdsh csm_comp "systemctl start csmd-compute" +xdsh ${UTILITY} "systemctl start csmd-utility" +xdsh ${COMPUTE_NODES} "systemctl start csmd-compute"