clusterbuster-connstat

#!/bin/bash

declare jq_cmd
read -r -d '' jq_cmd <<\EOF
[foreach .items[]? as $item
 ([[],[]];0;
 (if ($item.kind == "Pod") then
   ([foreach $item.spec.containers[]? as $container ([[],[]];0; [$item.metadata.namespace, $item.metadata.name, $container.name,
     (if ($item.status.containerStatuses? == null) then
      ($item.status.phase + ": " + $item.status.conditions[0].reason?)
     else
      [foreach $item.status.containerStatuses[]? as $status ([[],[]]; 0; if ($container.name == $status.name) then $status.state | keys[0] else null end)] end)] | flatten | map (select (. != null)) | join(" "))])
 else null end))] | flatten | map (select (. != null))[]
EOF

declare -i parallel=1
declare -i summarize=0
declare -i timeout=-1
declare -i verbose=0
declare basename=clusterbuster
declare -A expected_rows=()
declare -A cached_stats=()
declare statsfile=

declare OC
OC=$(type -p oc)
OC=${OC:-$(type -p kubectl)}	# kubectl might not work, though...

while getopts ':p:B:st:f:vq' opt ; do
    case "$opt" in
	B) basename=$OPTARG 	  ;;
	p) parallel=$OPTARG 	  ;;
	t) timeout=$OPTARG  	  ;;
	s) summarize=1	    	  ;;
	f) statsfile=$OPTARG	  ;;
	v) verbose=$((verbose+1)) ;;
	q) verbose=0        	  ;;
	*) ;;
    esac
done

(( parallel <= 0 )) && parallel=1

trap 'wait; exit' INT EXIT

function prettyprint() {
    local -a colwidths
    local -a rows
    local -a rowfields
    local -i i
    local f

    local -a rowzero=(+ NAMESPACE POD + CONTAINER STATE REF FAIL BLOCKS + CREATED START + CONNECTED DSTART FINISH TRANSFERTIME USER SYSTEM DATA XFERTIME RATE SLEEPMEAN SLEEPSTDEV)
    local -a formats=(.0s s s .0s s s d d d .0s .3f .3f .0s .3f .3f .3f .6f .3f .3f d .3f .3f .6f .6f)
    local -ri nfields=$((${#rowzero[@]} - 1))

    for i in $(seq 0 "$nfields") ; do
	f=${rowzero[$i]}
	f=${f%+}
	colwidths[$i]=${#f}
    done

    readarray -t rows

    for row in "${rows[@]}" ; do
	# shellcheck disable=SC2206
	rowfields=($row)
	for i in $(seq 0 $((${#rowfields[@]} - 1)) ) ; do
	    if (( ${colwidths[$i]} > 0 && ${#rowfields[$i]} > ${colwidths[$i]} )) ; then
		colwidths[$i]=${#rowfields[$i]}
	    fi
	done
    done

    local printfstr0=
    local printfstr1=

    local -i width

    for i in $(seq 0 "$nfields") ; do
	f=${rowzero[$i]}
	width=${colwidths[$i]}
	fmt=${formats[$i]}
	if [[ -n $printfstr0 && $printfstr0 != *'.0s' ]] ; then
	    printfstr0+=' '
	    printfstr1+=' '
	fi
	printfstr0+="%-${width}.${width}s"

	if [[ $fmt = *'s' ]] ; then
	    printfstr1+="%-${width}$fmt"
	else
	    printfstr1+="%${width}$fmt"
	fi
    done
    printfstr0+='\n'
    printfstr1+='\n'

    # shellcheck disable=SC2059
    printf "$printfstr0" "${rowzero[@]}"
    for row in "${rows[@]}" ; do
	# shellcheck disable=SC2059
	# shellcheck disable=SC2086
	printf "$printfstr1" $row
    done
}

function process_summary() {
    local -i line_count=0
    local -i total=0
    local -i first_start=0
    local -i last_end=0
    local -i total_et=0
    local -i total_cpu=0
    local -i total_cpu_util=0
    local -i total_iterations=0
    local -i pod_start=0
    local -i pod_first_start=0
    local -i pod_last_start=0
    local -i read_ops=0
    local -i write_ops=0
    local -i fsync_ops=0
    local -i readrate=0
    local -i writerate=0
    local -a output
    local mode=
    while read -r -a output ; do
	[[ -n "${output[*]}" ]] || continue
	if [[ -z "$mode" ]] ; then
	    case "${output[2]}" in
		*'soaker'*)   mode=soaker   ;;
		*'client'*)   mode=server   ;;
		*'sysbench'*) mode=sysbench ;;
		*)                          ;;
	    esac
	fi
	local -i start
	local -i end
	local -i et
	if (( verbose )) ; then
	    echo "${output[*]}"
	fi
	case "$mode" in
	    server)
		total+=${output[19]}
		start=$(clean "${output[14]}")
		end=$(clean "${output[15]}")
		et=$(clean "${output[20]}")
		if (( start < first_start || first_start == 0 )) ; then first_start=start; fi
		if (( end > last_end || last_end == 0 )) ; then last_end=end; fi
		;;
	    soaker)
		pod_start=$(($(clean "${output[12]}") - $(clean "${output[11]}")))
		if (( pod_start < pod_first_start || pod_first_start == 0 )) ; then pod_first_start=pod_start; fi
		if (( pod_start > pod_last_start )) ; then pod_last_start=pod_start; fi
		start=$(clean "${output[13]}")
		end=$(clean "${output[15]}")
		et=$(clean "${output[14]}")
		if (( start < first_start || first_start == 0 )) ; then first_start=start; fi
		if (( end > last_end || last_end == 0 )) ; then last_end=end; fi
		total_cpu+=$(clean "${output[16]}")
		total_cpu_util+=$(clean "${output[17]}")
		total_iterations+=$(clean "${output[18]}")
		;;
	    sysbench)
		start=$(clean "${output[11]}")
		end=$(clean "${output[13]}")
		et=$(clean "${output[19]}")
		if (( start < first_start || first_start == 0 )) ; then first_start=start; fi
		if (( end > last_end || last_end == 0 )) ; then last_end=end; fi
		read_ops+=$(clean "${output[14]}")
		write_ops+=$(clean "${output[15]}")
		fsync_ops+=$(clean "${output[16]}")
		readrate+=${output[17]}
		writerate+=${output[18]}
		;;
	esac
	line_count+=1
	total_et+=$et
    done
    (( line_count )) || return
    case "$mode" in
	server)
	    echo "Bytes: $total"
	    echo "Containers: $line_count"
	    echo "Average_elapsed: $(bc <<< "scale=3; $total_et/$line_count/1000")"
	    echo "Elapsed_span: $(bc <<< "scale=3; ($last_end-$first_start)/1000")"
	    echo "Data rate: $((total*2*1000/(total_et/line_count)/1024/1024))"
	    ;;
	soaker)
	    echo "Iterations: $total_iterations"
	    echo "Average_elapsed: $(bc <<< "scale=3; $total_et/$line_count/1000")"
	    echo "Elapsed_span: $(bc <<< "scale=3; ($last_end-$first_start)/1000")"
	    echo "First start: $(bc <<< "scale=3; ($pod_first_start)/1000")"
	    echo "Starting span: $(bc <<< "scale=3; ($pod_last_start-$pod_first_start)/1000")"
	    echo "Total CPU: $(bc <<< "scale=3; $total_cpu/1000")"
	    echo "CPU utilization: $(bc <<< "scale=3; $total_cpu_util/1000")%"
	    if (( total_cpu == 0 )) ; then
		echo "Iterations/CPU sec: 0"
	    else
		echo "Iterations/CPU sec: $(bc <<< "scale=0; $total_iterations*1000/$total_cpu")"
	    fi
	    if (( total_et == 0 )) ; then
		echo "Iterations/sec: 0"
	    else
		echo "Iterations/sec: $(bc <<< "scale=0; $total_iterations*1000*$line_count/$total_et")"
	    fi
	    ;;
	sysbench)
	    echo "Read ops/sec:  $(bc <<< "scale=3; $read_ops / $total_et * $line_count * 1000")"
	    echo "Write ops/sec: $(bc <<< "scale=3; $write_ops / $total_et * $line_count * 1000")"
	    echo "Fsync ops/sec: $(bc <<< "scale=3; $fsync_ops / $total_et * $line_count * 1000")"
	    echo "Read rate MB/sec: $(bc <<< "scale=3; $readrate / $total_et * $line_count")"
	    echo "Write rate MB/sec: $(bc <<< "scale=3; $writerate / $total_et * $line_count")"
	    ;;
	*)
	    echo "Unknown mode $mode"
	    ;;
    esac
}

function retrieve_pod_stats() {
    local ns=$1
    local pod=$2
    local container=$3
    local status=$4
    local -i connection_failures=0
    local -i connection_refused=0
    local -i blocks=0
    local statline=''
    local -a statlines=()
    local row_prefix='!'
    local pname="$ns $pod $container"
    case "$status" in
	running|completed|terminated)
	    # This is actually faster than reading lines if there's a lot of data.
	    local data
	    data="$(oc logs -n "$ns" "$pod" -c "$container")"
	    connection_refused=$(grep -c 'Connection refused' <<< "$data")
	    connection_failures=$(grep -c 'Could not connect' <<< "$data")
	    blocks=$(grep -c 'Write/Read' <<< "$data")
	    readarray -t statlines <<< "$(grep -E '^STATS' <<< "$data")"
	    if [[ $data = *"FINIS"* ]] ; then
		row_prefix=
	    fi
	    ;;
	'')
	    return
	    ;;
	*)
	    ;;
    esac
    if (( ${#statlines[@]} )) ; then
	local statline=
	for statline in "${statlines[@]}" ; do
	    echo "${row_prefix}-n $ns $pod -c $container $status $connection_failures $connection_refused $blocks $statline"
	done
    else
	echo "!-n $ns $pod -c $container $status $connection_failures $connection_refused $blocks"
    fi
}

declare wait_n=$(( BASH_VERSINFO[0] > 4 || (BASH_VERSINFO[0] > 4 && BASH_VERSINFO[1] >= 3) ))

function get_all_pods() {
    local basename=$1
    local ns
    while read -r ns ; do
	ns=${ns##*/}
	oc get pods -ojson -l "${basename}-client" -n "$ns"
    done <<< "$(oc get ns -l "$basename" --no-headers | awk '{print $1}')"
}

trap exit INT

function process_stats() {
    local -i curjobs=0
    local ns
    local pod
    local container
    local status
    while read -r ns pod container status ; do
	local xcache=${cached_stats["$ns $pod $container"]:-}
	if [[ -n "$xcache" ]] ; then
	    echo "$xcache"
	    continue
	fi
	if (( parallel > 1 )) ; then
	    if (( wait_n )) ; then
		while (( curjobs >= parallel )) ; do
		    wait -n
		    curjobs=$((curjobs - 1))
		done
	    elif (( curjobs >= parallel )) ; then
		wait
		curjobs=0
	    fi
	    retrieve_pod_stats "$ns" "$pod" "$container" "$status"&
	    curjobs=$((curjobs + 1))
	else
	    retrieve_pod_stats "$ns" "$pod" "$container" "$status"
	fi
    done

    if (( curjobs )) ; then
	wait
    fi
}

function get_stats() {
    process_stats <<< "$(get_all_pods "$basename" | jq -r "$jq_cmd")"
}

function clean() {
    local num=${1//./}
    while [[ $num = "0"* ]] ; do
	num=${num##0}
    done
    echo "${num:-0}"
}

function do_summarize() {
    # shellcheck disable=SC2155
    local -i start_time=$(printf '%(%s)T' -1)
    local pass=1
    while : ; do
	local -i found_all=1
	while read -r -a output ; do
	    [[ -n "${output[*]}" ]] || continue
	    local item="${output[1]} ${output[2]} ${output[4]}"
	    if [[ -z "${output[10]:-}" || ${output[0]} = '!'* ]] ; then
		found_all=0
	    else
		[[ -n "${cached_stats[$item]:-}" ]] || cached_stats[$item]=
		cached_stats[$item]+="${output[*]}"$'\n'
		unset pod_cache["$item"]
	    fi
	done <<< "$(get_stats)"
	if ((found_all)) ; then
	    echo "$(IFS=$'\n'; echo "${cached_stats[*]}")"
	    return
	fi
	pass=$((pass+1))
	if (( timeout > 0 && $(printf '%(%s)T' -1) - start_time > timeout  )) ; then
	    echo "Summary timed out ($timeout seconds)" 1>&2
	    return 1
	fi
    done
}

if (( summarize )) ; then
    case "$statsfile" in
	-)  process_summary		   ;;
	'') do_summarize | process_summary ;;
	*)  process_summary < "$statsfile" ;;
    esac
else
    get_stats |sed -e 's/^!//'
fi