monitor-pod-status

#!/bin/bash

declare -i delay="${1:-60}"
declare -i start
declare -i end
declare -A podstatus
declare -A nsstatus
declare -A secstatus
declare -A nodestatus
declare -i podcount
declare -i nscount
declare -i nodestatus

declare -r OC=${OC:-oc}

while :; do
    start=$(printf "%(%s)T" -1)
    podcount=0
    nscount=0
    nodecount=0
    podstatus=()
    nsstatus=()
    secstatus=()
    nodestatus=()
    # shellcheck disable=SC2034
    while read -r ns pod count status rest ; do
	[[ -n $pod ]] || break
	case "$status" in
	    Completed)		  status=C ;;
	    ContainerCreating)    status=O ;;
	    CrashLoopBackOff)     status=E ;;
	    Error)                status=E ;;
	    CreateContainerError) status=X ;;
	    Terminating)          status=T ;;
	    Pending)              status=P ;;
	    Running)              status=R ;;
	    ImagePullBackOff)	  status=I ;;
	    ErrImagePull)	  status=I ;;
	    *)                    status=U ;;
	esac
	podcount=$((podcount+1))
	podstatus[$status]=$((${podstatus[$status]:-0}+1))
    done <<< "$("$OC" get pods --all-namespaces -l clusterbuster=true --no-headers --ignore-not-found=true)"
    while read -r ns status rest ; do
	[[ -n $ns ]] || break
	status=${status:0:1}
	nscount=$((nscount+1))
	nsstatus[$status]=$((${nsstatus[$status]:-0}+1))
    done <<< "$("$OC" get namespaces -l clusterbuster=true --no-headers --ignore-not-found=true 2>/dev/null)"
    # shellcheck disable=SC2034
    while read -r ns name stype rest ; do
	[[ -n $ns ]] || break
	if [[ $stype == Opaque ]] ; then
	    stype=A
	else
	    stype=S
	fi
	secstatus[$stype]=$((${secstatus[$stype]:-0}+1))
	# We're looking for other secrets that can't be tagged.
    done <<< "$("$OC" get secrets --all-namespaces --no-headers --ignore-not-found=true 2>/dev/null |grep clusterbuster)"
    while read -r node status rest ; do
	[[ -n $node ]] || break
	status=${status:0:1}
	nodecount=$((nodecount+1))
	nodestatus[$status]=$((${nodestatus[$status]:-0}+1))
    done <<< "$("$OC" get nodes -l node-role.kubernetes.io/worker --no-headers --ignore-not-found=true 2>/dev/null)"
    printf "%s " "$(printf "%(%T)T" -1)" Pods: "$(for s in P R C O T E X I U ; do echo -n "${podstatus[$s]:-0}$s "; done)" "/ $podcount" "  NS:" "$(for s in A T ; do echo -n "${nsstatus[$s]:-0}$s "; done)" "/ $nscount" "  Sec:" "$(for s in A S ; do echo -n "${secstatus[$s]:-0}$s "; done)" "  Node:" "$(for s in R N ; do echo -n "${nodestatus[$s]:-0}$s "; done)" "/ $nodecount"
    echo
    end=$(printf "%(%s)T" -1)
    stime=$((start+delay-end))
    if (( stime > 0 )) ; then
	sleep "$stime"
    fi
done