experiment

#!/usr/bin/env zsh

if (( ${+PROJECT_DIR} )); then
   local project_dir=${PROJECT_DIR}
else
   local project_dir=$(pwd)
fi
RUN=${project_dir}/run
SENDER=rtpsctp/rtpsctpsend
RECEIVER=rtpsctp/rtpsctprecv
TMP_DIR=/tmp
TMP_RES_DIR=${TMP_DIR}/gst-sctp-results

TARGET=${project_dir}/build
TARGET_DEBUG=${project_dir}/build_debug

DEBUG_SENDER="sctpsink:4,rtpsctpsend:4,multiudpsink:1"
DEBUG_RECEIVER="sctpsrc:4,rtpsctprecv:4,udpsrc:1"

TRAPINT() {
   print "Caught SIGINT, aborting."
   kill ${${(v)jobstates##*:*:}%=*}
   ${=receiver_root} killall -9 rtpsctprecv
   ${=receiver_root} killall -9 tshark
   ssh root@${ip_sender} 'killall rtpsctpsend'
   wait

   [[ ! ${+verbose[1]} ]] && set +x
   rmdir .receiver_running.lok
   rmdir .receiver_start.lok

   sudo ip link set $phy2 up
   ssh root@${ip_sender} "tc qdisc replace dev $phy1 root netem loss 0% delay 0ms"
   ssh root@${ip_sender} "tc qdisc replace dev $phy2 root netem loss 0% delay 0ms"
   ${=receiver_root} tc qdisc replace dev $phy1 root netem loss 0% delay 0ms
   ${=receiver_root} tc qdisc replace dev $phy2 root netem loss 0% delay 0ms
   exit 1
}

# TRAPEXIT() {
   # return -1
# }

zparseopts -D -E -- R=remote v=verbose n:=num_runs d=debug -ntp=ntp W=nowrite C=capture -commit=commit P=print -pf=path_failure

if (( ${+verbose[1]} )); then
   set -x
fi

if (( ${+debug[1]} )); then
     DEBUG_SENDER="3,GST_REGISTRY:3,task:3,gstutils:5,GST_STATES:2,structure:3,basesink:3,rtpsctpsend:6,rtpbasepayload:3,rtpvrawpay:4,sctpsink:6,sctputils:4,usrsctpsnd:8,multiudpsink:3"
     # DEBUG_SENDER="2"
   DEBUG_RECEIVER="3,GST_REGISTRY:3,task:3,gstutils:5,GST_STATES:2,basesrc:3,pushsrc:3,rtpbasedepayload:3,rtpjitterbuffer:3,rtpsctprecv:5,sctpsrc:5,sctputils:4,usrsctprecv:8,udpsrc:3"
   # DEBUG_RECEIVER="4,GST_REGISTRY:4,task:4,gstutils:5,GST_STATES:3,basesrc:4,pushsrc:5,rtpbasedepayload:5,rtpjitterbuffer:4,rtpsctprecv:5,sctpsrc:5,sctputils:4,usrsctprecv:8,udpsrc:5,rtpmux:5"
   # DEBUG_RECEIVER="5"
   TARGET=$TARGET_DEBUG
fi

if (( ${+remote[1]} )); then
   receiver_root="ssh root@${ip_receiver}"
   receiver_run="ssh ibk@${ip_receiver}"
else
   receiver_root=sudo
   receiver_run=""
fi

local ip_sender=${IP_SENDER:-192.168.153.132}
local ip_receiver=${IP_RECEIVER:-192.168.153.135}
# ip a | grep -q ${ip_receiver}
# if (( $? )); then
#    echo "wrong receiver ip"
#    exit
# fi

local phy1=${PHY1:-ens37}
local phy2=${PHY2:-ens38}

local deadline=${DEADLINE:-450}
local num=${NUM:-2000}
local padding=${PADDING:-60}
local runs=${num_runs[2]:-1}
local pf_wait=${PF_WAIT:-2}

if (( ${+VARIANT} )); then
   local -a testing_variants=(${=VARIANT})
else
   # typeset -a testing_variants
   local -a testing_variants=(udp single cmt dupl dpr)
fi
if (( ${+DELAY} )); then
   local -a testing_delay=(${=DELAY})
else
   # typeset -a testing_delay
   local -a testing_delay=(5 20 40 60 80 100)
fi

if (( ${+path_failure[1]} )); then
   local -a testing_drop=(0)
elif (( ${+DROP} )); then
   local -a testing_drop=(${=DROP})
else
   # typeset -a testing_drop
   local -a testing_drop=(0 1 2 3 4 5 6 7 8 9 10 12 15 20)
fi

run_date=$(date +%F_%R)
if (( ! ${+nowrite[1]} )); then
   ${=receiver_run} mkdir -p ${project_dir}/results/${run_date}
   ${=receiver_run} cat > ${project_dir}/results/${run_date}/experiments_config <<EOL
project_dir=${project_dir}
ip_sender=${ip_sender}
ip_receiver=${ip_receiver}
phy1=${phy1}
phy2=${phy2}
variant=${testing_variants}
delay_on_link=${testing_delay}
deadline=${deadline}
num_frames=${num}
drop_rate=${testing_drop}
padding=${padding}
runs=${runs}
EOL
fi

all_count=1
(( all_runs = ${#testing_variants} * ${#testing_delay} * ${#testing_drop} * ${runs} ))
(( all_time = all_runs * ( num / 24 + 20) ))
date_finished=$(date +"%F %R" -d "+ ${all_time} sec")
echo "alltogether ${all_runs} runs, ${all_time}s, ETA: ${date_finished}"


for variant in ${testing_variants}; do
   for delay in ${testing_delay}; do
      for drop in ${testing_drop}; do
         run_id=${variant}_${delay}ms_${drop}d_${deadline}D_${num}n

         ${=receiver_run} rm -rf $TMP_RES_DIR
         ${=receiver_run} mkdir -p $TMP_RES_DIR
         ssh root@${ip_sender} "rm -rf ${TMP_RES_DIR} && mkdir -p ${TMP_RES_DIR}"


         if (( ${+ntp[1]} )); then
            ssh root@${ip_sender} "systemctl stop ntpd.service && ntpd -gq && systemctl start ntpd.service" &

            sudo systemctl stop ntpd.service
            sudo ntpd -gq
            sudo systemctl start ntpd.service

            wait
            # ntpdate ${ip_receiver}
            # if (( $? )); then
            #    echo "\033[0;31mNTP failed\033[0m"
            # fi
         fi

         count=1
         while (( $count <= $runs )) ; do
            local systime=$(date +%s)
            echo -e "\033[0;31mstarting variant ${variant} with delay= ${delay} drop_rate= ${drop}\033[0m"

            # sync so the receiver realy starts first (only for UDP)
            # [[ $variant =~ "udp" ]] &&
               mkdir .receiver_start.lok

            if (( ! ${+nowrite[1]} )); then
               receiver_tx_phy1_pre=$(cat /sys/class/net/${phy1}/statistics/tx_bytes)
               receiver_tx_phy2_pre=$(cat /sys/class/net/${phy2}/statistics/tx_bytes)
               receiver_rx_phy1_pre=$(cat /sys/class/net/${phy1}/statistics/rx_bytes)
               receiver_rx_phy2_pre=$(cat /sys/class/net/${phy2}/statistics/rx_bytes)
               sender_tx_phy1_pre=$(ssh root@${ip_sender} "cat /sys/class/net/${phy1}/statistics/tx_bytes")
               sender_tx_phy2_pre=$(ssh root@${ip_sender} "cat /sys/class/net/${phy2}/statistics/tx_bytes")
               sender_rx_phy1_pre=$(ssh root@${ip_sender} "cat /sys/class/net/${phy1}/statistics/rx_bytes")
               sender_rx_phy2_pre=$(ssh root@${ip_sender} "cat /sys/class/net/${phy2}/statistics/rx_bytes")
            fi

            echo "\033[0;31msetting delay= ${delay} drop_rate= ${drop}\033[0m"
            # simple gilbert p=${drop} > transission to bad stat (=100% drop)
            # k=75% > transission to good state (0% loss)
            ssh root@${ip_sender} "tc qdisc replace dev $phy1 root netem loss gemodel ${drop}% 75% delay ${delay}ms"
            ssh root@${ip_sender} "tc qdisc replace dev $phy2 root netem loss gemodel ${drop}% 75% delay ${delay}ms"
            ${=receiver_root} tc qdisc replace dev $phy1 root netem loss gemodel ${drop}% 75% delay ${delay}ms
            ${=receiver_root} tc qdisc replace dev $phy2 root netem loss gemodel ${drop}% 75% delay ${delay}ms

            (
            echo -e "\033[0;31m------- Receiver starting\033[0m"

            if (( ${+capture[1]} )); then
               [[ $variant =~ "udp" ]] && prot='udp' || prot='sctp'
               touch ${TMP_RES_DIR}/receiver_${count}.pcap
               chmod o+rw ${TMP_RES_DIR}/receiver_${count}.pcap
               sudo tshark -i ${phy2} -f "${prot} and src 192.168.0.2" -s 1500 -w ${TMP_RES_DIR}/receiver_${count}.pcap 2>&1 >/dev/null &
               tshark_pid=$!
               sleep 5
            fi

            # [[ $variant =~ "udp" ]] &&
               rmdir .receiver_start.lok
            [[ ! $variant =~ "udp" ]] && mkdir .receiver_running.lok

            if (( ${+print[1]} )); then
               ${=receiver_root} PROJECT_DIR=${project_dir} $RUN $TARGET $RECEIVER -d $DEBUG_RECEIVER -V $variant -D $deadline -T $systime --delay $delay --padding $padding 2>&1 | tee ${TMP_RES_DIR}/out_receiver_${count}
            else
               ${=receiver_root} PROJECT_DIR=${project_dir} $RUN $TARGET $RECEIVER -d $DEBUG_RECEIVER -V $variant -D $deadline -T $systime --delay $delay --padding $padding 2>&1 > ${TMP_RES_DIR}/out_receiver_${count}
            fi

            [[ ! $variant =~ "udp" ]] && rmdir .receiver_running.lok

            if (( ${+capture[1]} )); then
               sudo kill -INT ${tshark_pid}
            fi

            echo -e "\033[0;32m------- Receiver finished\033[0m"
            ) &
            receiver_pid=$!


            (
            while [ -d .receiver_start.lok ]; do sleep 1; done
            echo -e "\033[0;31m------- Sender starting\033[0m"

            if (( ${+print[1]} )); then
               ssh root@${ip_sender} "PROJECT_DIR=${project_dir} IP_RECEIVER=${ip_receiver} $RUN $TARGET $SENDER -d $DEBUG_SENDER -V $variant -D $deadline -n $num --delay $delay --padding $padding -T $systime" 2>&1 | tee ${TMP_RES_DIR}/out_sender_${count}
            else
               ssh root@${ip_sender} "PROJECT_DIR=${project_dir} IP_RECEIVER=${ip_receiver} $RUN $TARGET $SENDER -d $DEBUG_SENDER -V $variant -D $deadline -n $num --delay $delay --padding $padding -T $systime 2>&1 > ${TMP_RES_DIR}/out_sender_${count}"
            fi

            local c=0
            while [[ ! $variant =~ "udp" ]] && [ -d .receiver_running.lok ]; do 
               sleep 1
               (( c++ ))
               if (( $c >= 30 )); then
                  ${=receiver_root} killall rtpsctprecv
                  echo -e "\033[0;31mreceiver killed\033[0m"
                  touch ${TMP_RES_DIR}/receiver_killed_${count}
                  # wait $receiver_pid
                  break
               fi
            done

            if (( ! ${+nowrite[1]} )); then
            fi
            echo -e "\033[0;32m------- Sender finished\033[0m"
            ) &
            sender_pid=$!

            if (( ${+path_failure[1]} )); then
               while [ -d .receiver_start.lok ]; do sleep 1; done
               sleep $pf_wait
               sudo ip link set $phy2 down
               echo -e "\033[0;32m------- interface $phy2 taken down\033[0m"
            fi

            wait $sender_pid $receiver_pid

            if (( ${+path_failure[1]} )); then
               sudo ip link set $phy2 up
               while ! ip -o -4 a s $phy2 | grep "192.168" -q; do; done
               echo -e "\033[0;32m------- interface $phy2 brought up\033[0m"
            fi

            # set to 0ms delay and 0% loss..
            ssh root@${ip_sender} "tc qdisc replace dev $phy1 root netem loss 0% delay 0ms"
            ssh root@${ip_sender} "tc qdisc replace dev $phy2 root netem loss 0% delay 0ms"
            ${=receiver_root} tc qdisc replace dev $phy1 root netem loss 0% delay 0ms
            ${=receiver_root} tc qdisc replace dev $phy2 root netem loss 0% delay 0ms

            if (( ! ${+nowrite[1]} )); then
               local receiver_tx_phy1_post=$(cat /sys/class/net/${phy1}/statistics/tx_bytes)
               local receiver_tx_phy2_post=$(cat /sys/class/net/${phy2}/statistics/tx_bytes)
               local receiver_rx_phy1_post=$(cat /sys/class/net/${phy1}/statistics/rx_bytes)
               local receiver_rx_phy2_post=$(cat /sys/class/net/${phy2}/statistics/rx_bytes)
               ${=receiver_run} cat > ${TMP_RES_DIR}/receiver_transfered_bytes_${count} <<EOL
receiver_tx_phy1=$(( receiver_tx_phy1_post - receiver_tx_phy1_pre ))
receiver_tx_phy2=$(( receiver_tx_phy2_post - receiver_tx_phy2_pre ))
receiver_rx_phy1=$(( receiver_rx_phy1_post - receiver_rx_phy1_pre ))
receiver_rx_phy2=$(( receiver_rx_phy2_post - receiver_rx_phy2_pre ))
EOL

               sender_tx_phy1_post=$(ssh root@${ip_sender} "cat /sys/class/net/${phy1}/statistics/tx_bytes")
               sender_tx_phy2_post=$(ssh root@${ip_sender} "cat /sys/class/net/${phy2}/statistics/tx_bytes")
               sender_rx_phy1_post=$(ssh root@${ip_sender} "cat /sys/class/net/${phy1}/statistics/rx_bytes")
               sender_rx_phy2_post=$(ssh root@${ip_sender} "cat /sys/class/net/${phy2}/statistics/rx_bytes")
               ${=receiver_run} cat > ${TMP_RES_DIR}/sender_transfered_bytes_${count} <<EOL
sender_tx_phy1=$(( sender_tx_phy1_post - sender_tx_phy1_pre ))
sender_tx_phy2=$(( sender_tx_phy2_post - sender_tx_phy2_pre ))
sender_rx_phy1=$(( sender_rx_phy1_post - sender_rx_phy1_pre ))
sender_rx_phy2=$(( sender_rx_phy2_post - sender_rx_phy2_pre ))
EOL
            fi

            echo "\033[0;32mrun ${run_id} finished (${all_count}/${all_runs}) ETA: $(date +"%F %R" -d "+ $(( ( all_runs - all_count) * ( num / 24 + 10) )) sec") \033[0m"
            if (( ! ${+nowrite[1]} )); then
               ${=receiver_run} mv ${TMP_RES_DIR}/experiment_trace_in.csv ${TMP_RES_DIR}/experiment_trace_in_${count}.csv
               ${=receiver_run} mv ${TMP_RES_DIR}/experiment_trace_out.csv ${TMP_RES_DIR}/experiment_trace_out_${count}.csv

               if [[ ! $variant =~ "udp" ]]; then
                  ${=receiver_run} mv ${TMP_RES_DIR}/usrsctp_stats_receiver ${TMP_RES_DIR}/usrsctp_stats_receiver_${count}
                  scp -q root@${ip_sender}:${TMP_RES_DIR}/usrsctp_stats_sender ${TMP_RES_DIR}/usrsctp_stats_sender_${count}
               fi

               if (( ! ${+print[1]} )); then
                  scp -q root@${ip_sender}:${TMP_RES_DIR}/out_sender_${count} ${TMP_RES_DIR}/
               fi
            fi
            (( count++ ))
            (( all_count++ ))
         done

         if (( ! ${+nowrite[1]} )); then
            echo "\033[0;32mall runs finished ID: ${run_id} >> writing results to ${project_dir}/results/${run_date}/${run_id}\033[0m"
            ${=receiver_run} mkdir -p ${project_dir}/results/${run_date}/${run_id}
            ${=receiver_run} mv ${TMP_RES_DIR}/* ${project_dir}/results/${run_date}/${run_id}
            ${=receiver_run} cat > ${project_dir}/results/${run_date}/${run_id}/run_config <<EOL
project_dir=${project_dir}
ip_sender=${ip_sender}
ip_receiver=${ip_receiver}
phy1=${phy1}
phy2=${phy2}
variant=${variant}
delay_on_link=${delay}
deadline=${deadline}
num_frames=${num}
drop_rate=${drop}
padding=${padding}
runs=${runs}
EOL
         fi
         sleep 2

      done # drop
   done # delay
done # variant

if (( ${+commit[1]} )) && (( ! ${+nowrite[1]} )); then
   mkdir lab_results
   mv ${project_dir}/results/${run_date} lab_results/
   git add lab_results/${run_date}
   git commit -m "lab results: ${run_date}\nD:${deadline} n:${num_frames} d:${testing_delay} V:${testing_variants} dr:${testing_drop}%"
   git push
fi

[[ ! ${+verbose[1]} ]] && set +x
rmdir .receiver_running.lok 2>/dev/null
rmdir .receiver_start.lok 2>/dev/null
exit 0