clusterbuster

#!/bin/bash

# Copyright 2019-2020 Robert Krawitz/Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -u

declare -i namespaces=1
declare -i depsPerNamespace=1
declare -i secrets=0
declare -i replicas=1
declare -i parallel=1
declare -i firstDeployment=0
declare -i sleepSecrets=0
declare -i sleepNamespaces=0
declare -i sleepDeployments=0
declare -i parallelSecrets=0
declare -i parallelNamespaces=0
declare -i parallelDeployments=0
declare -i blocksize=1
declare -i blocksizeSecrets=0
declare -i blocksizeNamespaces=0
declare -i blocksizeDeployments=0
declare -i containers=1
declare -i createRoutes=0
declare -i sleeptime=0
declare -i doit=1
declare -i blockitemcount=0
declare -i port=7777
declare -i syncPort=7778
declare -i affinity=0
declare -i verbose=0
declare -i wait_for_secrets=1
declare dataRate=0
declare -i bytes=0
declare -i bytesMax=0
declare -i msgSize=32768
declare -i poddelay=0
declare -i xferTime=0
declare -i xferTimeMax=0
declare -i exitAtEnd=0
declare -i reportObjectCreation=1
declare -i defaultBytes=1000000000
declare -A objectsCreated=()
declare podType=ClusterBuster
declare podsleeptime=infinity
declare basename=clusterbuster
declare deploymentType=deployment
declare -i basetime
declare opt
declare -r nl=$'\n'
declare -a pod_resource_requests=()
declare -a pod_resource_limits=()
declare -a container_resource_requests=()
declare -a container_resource_limits=()
declare -A namespaces_in_use=()
declare -a namespaces_to_create=()
declare -i scale_ns=0
declare -i scale_deployments=1
declare -i syncStart=0
declare runtimeClass=
declare -a emptydirs=()
declare -a sysbench_generic_options=()
declare -a sysbench_fileio_options=()
declare -A sysbench_fileio_tests=()
declare -i report=0
declare -i verbose_report=0
declare -i cleanup=0
declare -i timeout=-1
declare pathdir=${0%/*}
declare -a unknown_opts=()
declare -i totalObjectsCreated=0

declare accumulateddata=

declare OC
OC=$(type -p oc)
OC=${OC:-$(type -p kubectl)}	# kubectl might not work, though...

function _helpmsg() {
    local opt
    for opt in "$@" ; do
	echo "Unknown option $opt"
    done
cat <<EOF
Usage: $0 [options] [name]
    Options:
       -B basename     Base name of pods (default clusterbuster).
                       All objects are labeled with this name.
       -b blocksize    Number of objects to create in one oc call (default 1)
       -c containers   Number of containers per pod
       -d deployments  Number of deployments per namespace (default 1)
       -h              Print this help message
       -H              Print extended help.  Use --help-everything to
                       get an extended description.
       -n              Print what would be done without doing it
       -N namespaces   Number of namespaces to create (default 1)
       -P podtype      Pod type (classic, pause, server, cpu, sysbench)
       -p parallel     Number of operations to perform in parallel (default 1)
       -r replicas     Number of replicas per deployment (default 1)
                       For client/server, this refers to client pods.
                       For cpu, this refers to processes per pod.
       -S              Scale up (add namespaces to existing load) (default no)
       -s secrets      Number of secrets per deployment (default 0)
       -T type         Use the specified deployment type:
                       pod or deployment (default $deploymentType)
       -w wait         Number of seconds to wait between ops (default 0)
       -W              Wait for secrets to be created (default 1)
       -z podsleep     Amount of time in seconds for pod to sleep before exit
                       (default infinity).  This may be an expression evaluated
                       as an argument to sleep(1) and need not be a numeric
                       constant.
       --opt[=val]     Set the specified option.  Use -H or --help-everything
                       to get a list.
       -Y              Sync start of data transfer or CPU load (default no)

    Client/server options:
       -D data rate    Data rate for client/server (MB/sec) (default 0)
                       May be a decimal number.
       -E              Don't exit after all data transmitted (default)
       -e              Exit after all data transmitted.
       -M MB[,max]     Amount of data to transfer.  If two values are
                       specified, the amount will be randomly chosen
                       per container.
       -m msgsize      Message size in data transfer
       -t time[,max]   Time (in seconds) to transfer data (default 0
                       means no time limit).  If two values are specified,
                       the amount will be randomly chosen per container.
       -v              Print verbose log messages.  Useful with
                       clusterbuster-connstat.
       -x              Use pod affinity between client and server (default no)
       -X              Use pod anti-affinity between client/server (default no)
EOF
}

function _helpExtended() {
    _helpmsg
    cat <<'EOF'

Extended Options:
    General Extended Options (short equivalents):
       --pod_type=type  Specify the type of pod (-P)
       --basename=name  Specify the base name (-B)
       --containers=N   Number of containers per pod (-c)
       --namespaces=N   Number of namespaces (-N)
       --secrets=N      Number of secrets (-s)
       --replicas=N     Number of replicas per deployment (-r)
       --podsleep=N     Time for pod to sleep before exit (-z)
       --report         Print report (client/server, soaker,
                        sysbench only)
       --verbose-report Print detailed report
       --cleanup        Clean up generated objects
       --timeout=N      Time out reporting

    Tuning object creation (short equivalents):
       --scale-ns=[0,1] Scale up the number of namespaces.
                        (default 0)
       --scale-deployments=[0,1]
                        Scale up the number of deployments.
                        (default 1)
       --first_deployment=N
                        Specify the index of the first deployment.
                        Default is 0 (but see --scale_deployments)
       --first_secret=N
       --first_namespace=N
       --first_deployment=N
       --blocksize=N    Number of objects per oc call (-b)
                        Below options default to blocksize
       --blocksize_secrets=N
       --blocksize_namespaces=N
       --blocksize_deployments=N
       --sleep=N        Number of seconds between object creations (-w)
                        Below options default to sleeptime
       --sleep_secrets=N
       --sleep_namespaces=N
       --sleep_deployments=N
       --parallel=N     Number of operations in parallel (-p)
                        Below options default to parallel
       --parallel_secrets=N
       --parallel_namespaces=N
       --parallel_deployments=N

    Pod Options:
       --<type>=<resource=value>
                        Resource requests and limits
                        <type> is pod_resource_request, pod_resource_limit,
                        container_resource_request, or container_resource_limit
       --runtimeclass=class
                        Run the pods in the designated runtimeclass.
       --kata           Synonym for --runtimeclass=kata-qemu
       --emptydir=dir	Mount an emptydir volume at the specified mount point.

    Client/server options:
       --affinity       Force affinity between client and server pods
                        in a deployment.
       --anti-affinity  Force anti-affinity between client and server pods
                        in a deployment.
       --bytes=N[,M]    Number of bytes to transfer
                        Also --kb, --kib, --mb, --mib, --gb, --gib
                        (-M is equivalent to --mb)
       --create_routes  Create routes for services (default 0)
       --datarate=N     Data transfer rate (-D)
       --exit_at_end    Exit upon completion of transfer (-e)
       --msgsize=N      Message size for data transfer (-m)
       --poddelay=N     Time in seconds to wait for pods to start data xfer
       --transfertime=N Time to transfer data (-t).  Two comma-separated
                        numbers may be used to specify maximum time.
       --verbose        Print verbose log messages (-v)
       --wait_secrets   Wait for secrets to be created (-W) (default 1)

    Sysbench General Options:
       --sysbench-general-options=<options>
                        Space or comma separated general Sysbench options

    Sysbench FileIO Options:
       --sysbench-fileio-tests=<modes>
                        Space of comma separated list of file test modes
                        to use (seqwr, seqrewr, seqrd, rndrd, rndwr, rndrw)
                        If multiple modes are specified, modes are
                        spread around.
       --sysbench-fileio-options=<options>
                        Space or comma separated fileio Sysbench options
EOF
}

function _helpEverything() {
    _helpExtended
    cat <<'EOF'


Clusterbuster is a tool to permit you to load a configurable workload
onto an OpenShift cluster.  This is not a full-featured workload
generator like clusterloader; it generates very specific types of load
that for the most part focus on API objects.

Clusterbuster allows you to create three kinds of pods (via
deployments).  The type of pod is selected by the -P option.

- A simple pause-type implementation written in perl that emits a log
  heartbeat once a minute.

- Pause pod (uses the least resources).  Useful for testing the control
  plane, but not the data plane.

- A client-server workload, with optional bidirectional data transfer
  at a specified data rate.  Allows load testing of the data plane.


All three types allow you to configure the following load parameters:

- Number of namespaces created (-N)

- Number of deployments per namespace (-d)

- Number of replicas per deployment (-r)

- Number of containers per replica (-c)

- Number of secrets per deployment (-s)

The total number of pods created is (N * d * r) and the total number
of containers is (N * d * r * c).  The number of secrets is (N * d).
This is useful for testing of scaling of both nodes and apiserver/etcd
combination on the control plane.


The client-server workload consists of one server pod, with one
container, per deployment.  The clients and server can optionally
engage in ping-pong data transfer.  The number of client pods is
controlled by the number of replicas.  Therefore, each server will
service (r * c) connections.  Each container will attempt to transmit
up to the specified data rate (-D) megabytes/second, and then receive
the amount of data.  The amount of data to be sent can be specified
via -M, in megabytes; the size of each message can be specified via
-m, in bytes.  By default, no data is sent.  The amount of time to
perform data transfer can be specified by -t (default is none).

Via -x and -X it is possible to specify whether clients will be
colocated with their server (-x) or forced onto different nodes (-X).
If neither option is specified, clusterbuster allows OpenShift to
place pods as it sees fit.  You can use `oc logs` on a client pod (or
use Prometheus metrics) to get data transfer information; with -v,
verbose information about each transfer is printed.

The client logs the following at the end of any data transfer:

STATS <atime> <stime> <gtime> <etime> <endtime> <user> <sys> <interval> <data> <detime> <rate> <mean> <stdev>

defined as:

atime           Time (seconds from start of run) of "oc apply" to
                create the client
stime           Time when the client started running
gtime           Time when "gethostbyname" completed
etime           Time when connection to server succeeded
endtime         Time when client completed
user            User CPU time
user            System CPU time
interval        Duration between first attempt to connect and
                successful connection
data            Number of data bytes sent and received
detime          Elapsed time for data transmission/reception
rate            Rate (MB/sec) of data transfer, if any
mean            Mean time sleeping between messages to attempt to
                achieve the desired transmission rate
stdev           Standard deviation of sleep interval between messages

You can use `clusterbuster-connstat` to extract client information about a
running or completed test run, including the above data.  Running
`clusterbuster -v` allows `clusterbuster-connstat` to extract data about
transfers completed thus far.


You can run multiple concurrent clusterbusters by choosing different
basenames for each instance (-B).  All objects created by that run
will be tagged with the basename, in addition to 'clusterbusterbase'
(to allow you to easily find or delete all objects created by
clusterbuster).


Examples:

- You have a 30 node cluster and wish to test 500 pods per node, all
  in one namespace:

  clusterbuster -d 15000

- You wish to use 1000 namespaces for the same test:

  clusterbuster -N 1000 -d 15

- You wish to add 150,000 secrets to further stress the apiserver and
  etcd:

  clusterbuster -N 1000 -d 15 -s 10

- You wish to exchange data at 10 MB/sec with a total of 2 GB per client,
  with 5 deployments in 2 namespaces each of which uses 4 replica clients
  and 3 containers.  You wish to place the clients on nodes other than the
  servers:

  clusterbuster -P server -N 2 -d 5 -r 4 -c 3 -X -v

EOF
}

function help() {
    _helpmsg "$@" | "${PAGER:-more}"
    exit 1
}

function help_extended() {
    _helpExtended | "${PAGER:-more}"
    exit 1
}

function help_everything() {
    _helpEverything | "${PAGER:-more}"
    exit 1
}

function ts() {
    local dt
    dt=$(date '+%s.%N')
    local sec=${dt%.*}
    local ns=${dt#*.}
    echo "${sec}.${ns:0:6}"
}

function timestamp() {
    while read -r 'LINE' ; do
	printf "%s %s\n" "$(TZ=GMT-0 date '+%Y-%m-%dT%T.%N' | cut -c1-26)" "$LINE"
    done
}

trap exit INT

function find_free_port_perl() {
    cat <<'EOF'
#!/usr/bin/perl

use Socket;
use POSIX;
use strict;
use Getopt::Long;
Getopt::Long::Configure("bundling", "pass_through");

my ($port) = 4097;

GetOptions("p=i" => \$port);

while ($port < 65535) {
    my $sockaddr = "S n a4 x8";
    my ($socket);
    socket($socket, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
    setsockopt($socket,SOL_SOCKET, SO_REUSEADDR, pack("l",1)) || die "setsockopt reuseaddr: $!\n";
    setsockopt($socket,SOL_SOCKET, SO_KEEPALIVE, pack("l",1)) || die "setsockopt keepalive: $!\n";
    if (bind($socket, pack($sockaddr, AF_INET, $port, "\0\0\0\0"))) {
	print "$port\n";
	exit 0;
    }
    $port++;
}
die "Can't find free port!\n";
EOF
}

function find_free_port() {
    perl -e "$(find_free_port_perl)"
}

function get_logs_perl() {
    cat <<'EOF'
#!/usr/bin/perl

use Socket;
use POSIX;
use strict;
use Getopt::Long;
Getopt::Long::Configure("bundling", "pass_through");

my ($timeout) = -1;
my ($port) = 4096;
my ($expect_count) = -1;

GetOptions("t=i" => \$timeout,
	   "p=i" => \$port,
	   "c=i" => \$expect_count);

my $sockaddr = "S n a4 x8";
my ($socket);
socket($socket, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
setsockopt($socket,SOL_SOCKET, SO_REUSEADDR, pack("l",1)) || die "setsockopt reuseaddr: $!\n";
setsockopt($socket,SOL_SOCKET, SO_KEEPALIVE, pack("l",1)) || die "setsockopt keepalive: $!\n";
if (!bind($socket, pack($sockaddr, AF_INET, $port, "\0\0\0\0"))) {
    die "Can't bind to port $port\n";
}
listen($socket, 100) || die "Can't listen on port $port\n";

my ($client_count) = 0;
eval {
    if ($timeout > 0) {
	alarm $timeout;
    }
    my $kid;
    while ($expect_count < 0 || $client_count < $expect_count) {
    	my $child = fork;
	if ($child == 0) {
	    my ($client);
	    accept ($client, $socket) || next;
	    close ($socket);
	    my ($tbuf) = "";
	    my ($status);
	    if (($status = sysread($client, $tbuf, 1024)) > 0) {
		print "$tbuf\n";
	    }
	    exit(0);
        }
	do {
	    $kid = waitpid(-1, WNOHANG);
 	} while ($kid > 0);
	$client_count++;
    }
    do {
	$kid = wait();
    } while ($kid > 0);
};

if ($@) {
    die "$@";
}
EOF
}

function get_logs() {
    local varg=
    ((verbose_report)) && varg=-v
    exec perl -e "$(get_logs_perl)" -- "$@" | "${pathdir:-.}/clusterbuster-connstat" -s -f - "$varg"
}

function set_xfer() {
    local sizespec=$1
    local -i scale=${2:-1}
    if [[ $sizespec = *','* ]] ; then
	bytes=$((${sizespec#*,} * scale))
	bytesMax=$((${sizespec%%,*} * scale))
	if (( bytes > bytesMax )) ; then
	    local -i tmp=$bytes
	    bytes=$bytesMax
	    bytesMax=$tmp
	fi
    else
	bytes=$((sizespec * scale))
	bytesMax=$((sizespec * scale))
    fi
}

function set_xfer_time() {
    local timespec=$1
    if [[ $timespec = *','* ]] ; then
	xferTime=${timespec#*,}
	xferTimeMax=${timespec%%,*}
	if (( xferTime > xferTimeMax )) ; then
	    local -i tmp=$xferTime
	    xferTime=$xferTimeMax
	    xferTimeMax=$tmp
	fi
    else
	xferTime=$timespec
	xferTimeMax=$timespec
    fi
}

function process_option() {
    local option=$1
    local optname
    local optvalue
    optname=${option%%=*}
    optname=${optname,,}
    optvalue=${option#*=}
    noptname=${optname//-/_}
    if [[ $option != *'='* ]] ; then
	if [[ $optname = "no_"* || $optname = "dont_"* ]] ; then
	    optname=${optname#dont_}
	    optname=${optname#no_}
	    optvalue=0
	else
	    optvalue=1
	fi
    fi
    # shellcheck disable=SC2206
    case "$noptname" in
	help_all*)		    help_extended				;;
	help_everything*)	    help_everything				;;
	help*)			    help					;;
	pod_type)		    podType=$optvalue				;;
	basename)		    basename=$optvalue				;;
	blocksize)		    blocksize=$optvalue				;;
	containers)		    containers=$optvalue			;;
	namespaces)		    namespaces=$optvalue			;;
	datarate)		    dataRate=$optvalue				;;
	bytes)			    set_xfer "$optvalue"			;;
	kb|kilobytes)		    set_xfer "$optvalue" 1000			;;
	kib|kibibytes)		    set_xfer "$optvalue" 1024			;;
	mb|megabytes)		    set_xfer "$optvalue" 1000000		;;
	mib|mebibytes)		    set_xfer "$optvalue" 1048576		;;
	gb|gigabytes)		    set_xfer "$optvalue" 1000000000		;;
	gib|gibibytes)		    set_xfer "$optvalue" 1073741824		;;
	msgsize)		    msgSize=$optvalue				;;
	secrets)		    secrets=$optvalue				;;
	verbose)		    verbose=$optvalue				;;
	quiet)			    verbose=$((!optvalue))			;;
	replicas)		    replicas=$optvalue				;;
	parallel)		    parallel=$optvalue				;;
	sleep)			    sleeptime=$optvalue				;;
	podsleep)		    podsleeptime=$optvalue			;;
	poddelay)		    poddelay=$optvalue				;;
	transfertime)		    set_xfer_time "$optvalue"			;;
	first_deployment)	    firstDeployment=$optvalue			;;
	parallel_secrets)	    parallelSecrets=$optvalue			;;
	parallel_namespaces)	    parallelNamespaces=$optvalue		;;
	parallel_deployments)	    parallelDeployments=$optvalue		;;
	blocksize_secrets)	    blocksizeSecrets=$optvalue			;;
	blocksize_namespaces)	    blocksizeNamespaces=$optvalue		;;
	blocksize_deployments)	    blocksizeDeployments=$optvalue		;;
	sleep_secrets)		    sleepSecrets=$optvalue			;;
	sleep_namespaces)	    sleepNamespaces=$optvalue			;;
	sleep_deployments)	    sleepDeployments=$optvalue			;;
	exit_at_end)		    exitAtEnd=$optvalue				;;
	pod_resource_request)	    pod_resource_requests+=("$optvalue")	;;
	pod_resource_limit)	    pod_resource_limits+=("$optvalue")		;;
	container_resource_request) container_resource_requests+=("$optvalue")	;;
	container_resource_limit)   container_resource_limits+=("$optvalue")	;;
	create_routes)		    createRoutes=$optvalue			;;
	wait_secrets)		    wait_for_secrets=$optvalue			;;
	scale_ns)	   	    scale_ns=$optvalue				;;
	scale_deployments)   	    scale_deployments=$optvalue			;;
	deployment_type)            deploymentType=$optvalue			;;
	runtimeclass|runtime_class) runtimeClass=$optvalue			;;
	report_object_creation)	    reportObjectCreation=$optvalue		;;
	kata)			    runtimeClass=kata				;;
	report)	    		    report=$optvalue				;;
	cleanup)		    cleanup=$optvalue				;;
	timeout)		    timeout=$optvalue				;;
	verbose_report)		    verbose_report=$optvalue			;;
	emptydir)		    emptydirs+=("$optvalue")			;;
	sync|syncstart|sync_start)  syncStart=$((1-syncStart))			;;
	sysbench_general_options)   sysbench_general_options+=(${optvalue//,/ });;
	sysbench_fileio_options)    sysbench_fileio_options+=(${optvalue//,/ }) ;;
	sysbench_fileio_tests)
	    local ftest
	    for ftest in ${optvalue//,/ } ; do
		sysbench_fileio_tests[$ftest]=1
	    done
	    ;;
	affinity)
	    case "$optvalue" in
		1|'') affinity=1      ;;
		2|anti) affinity=2    ;;
		*) affinity=0         ;;
	    esac
	    ;;
	antiaffinity|anti_affinity)
	    case "$optvalue" in
		1|'') affinity=2      ;;
		*) affinity=0         ;;
	    esac
	    ;;
	*) unknown_opts+=("$noptname") ;;
    esac
    if [[ $deploymentType != pod && $deploymentType != deployment ]] ; then
	echo "-T/--deployment_type must be either por or deployment"
	help
    fi
}

while getopts ":B:b:c:D:d:EeHhM:m:N:nP:p:Qqr:Ss:T:t:vWw:XxYz:-:" opt ; do
    case "$opt" in
	B) basename="$OPTARG"		;;
	D) dataRate="$OPTARG"           ;;
	E) exitAtEnd=0			;;
	e) exitAtEnd=1			;;
	h) help				;;
	H) help_extended		;;
	M) set_xfer "$OPTARG" 1000000	;;
	N) namespaces="$OPTARG"         ;;
	P) podType="$OPTARG"		;;
	X) affinity=2			;;
	b) blocksize="$OPTARG"          ;;
	c) containers="$OPTARG"         ;;
	d) depsPerNamespace="$OPTARG"   ;;
	m) msgSize="$OPTARG"		;;
	n) doit=0			;;
	p) parallel="$OPTARG"           ;;
	Q) reportObjectCreation=0	;;
	q) verbose=0			;;
	r) replicas="$OPTARG"           ;;
	S) scale_ns=1			;;
	s) secrets="$OPTARG"            ;;
	T) deploymentType="$OPTARG"	;;
	t) set_xfer_time "$OPTARG"	;;
	v) verbose=1			;;
	W) wait_for_secrets=0		;;
	w) sleeptime="$OPTARG"          ;;
	x) affinity=1			;;
	Y) syncStart=$((1-syncStart))	;;
	z) podsleeptime="$OPTARG"	;;
	-) process_option "$OPTARG"	;;
	*) help "$OPTARG"               ;;
    esac
done
[[ -n "${unknown_opts[*]}" ]] && help "${unknown_opts[@]}"

(( !parallelSecrets )) && parallelSecrets=$parallel
(( !parallelNamespaces )) && parallelNamespaces=$parallel
(( !parallelDeployments )) && parallelDeployments=$parallel

(( !blocksizeSecrets )) && blocksizeSecrets=$blocksize
(( !blocksizeNamespaces )) && blocksizeNamespaces=$blocksize
(( !blocksizeDeployments )) && blocksizeDeployments=$blocksize

[[ -n $runtimeClass ]] && runtimeClass="runtimeClassName: \"$runtimeClass\""

shift $((OPTIND - 1))

if [[ -z $OC && $doit -gt 0 ]] ; then
    echo "Cannot find oc or kubectl command, exiting!"
    exit 1
fi

if (( msgSize <= 0 )) ; then
    echo "Message size must be positive, exiting!"
    exit 1
fi

if [[ $dataRate != 0 && $dataRate != '' && $xferTimeMax -eq 0 && bytesMax -eq 0 ]] ; then
    bytes=$defaultBytes
    bytesMax=$defaultBytes
fi

case "$podType" in
    classic|ClusterBuster) podType=ClusterBuster ;;
    server)                podType=Server        ;;
    cpu*|CPU*|soaker)      podType=CPUSoaker     ;;
    sysbench|Sysbench)     podType=Sysbench      ;;
    pause|PausePod|*)      podType=PausePod      ;;
esac

if [[ -n ${1:-} ]] ; then
    basename="$1"
    shift
fi

[[ -z "$*" ]] || echo "Warning: extraneous arguments $# after basename will be ignored!" 1>&2

function indent_1() {
    local -i column="$1"
    local line
    while IFS='' read -r 'line' ; do
	[[ -z "$line" ]] || printf "%${column}s%s\n" ' ' "$line"
    done
}

function indent() {
    local -i column="$1"
    shift
    if [[ -n "$*" ]] ; then
	# "$@" | indent_1 "$column" strips whitespace with bash 4.2
	indent_1 "$column" <<< "$("$@")"
    else
	indent_1 "$column"
    fi
}

function finishCreation() {
    if [[ $doit -ne 0 && -n $accumulateddata ]] ; then
	"${OC}" apply -f - <<< "$accumulateddata" | timestamp
	accumulateddata=
    fi
}

function createResources() {
    local token
    for token in "$@" ; do
	local resource=${token%%=*}
	local value=${token#*=}
	echo "$resource: $value"
    done
}

function createPodResources() {
    if (( ${#pod_resource_limits[@]} + ${#pod_resource_requests[@]} )) ; then
	echo "resources:"
	if (( ${#pod_resource_limits[@]} )) ; then
	    echo "  limits:"
	    indent 4 createResources "${pod_resource_limits[@]}"
	fi
	if (( ${#pod_resource_requests[@]} )) ; then
	    echo "  requests:"
	    indent 4 createResources "${pod_resource_requests[@]}"
	fi
    fi
}

function createContainerResources() {
    if (( ${#container_resource_limits[@]} + ${#container_resource_requests[@]} )) ; then
	echo "resources:"
	if (( ${#container_resource_limits[@]} )) ; then
	    echo "  limits:"
	    indent 4 createResources "${container_resource_limits[@]}"
	fi
	if (( ${#container_resource_requests[@]} )) ; then
	    echo "  requests:"
	    indent 4 createResources "${container_resource_requests[@]}"
	fi
    fi
}

function createObject() {
    local data=
    local line=
    while IFS='' read -r 'line' ; do
	if [[ -n $line ]] ; then
	    [[ -n $data ]] || data="---$nl"
	    data+="$line$nl"
	fi
    done
    data+="$nl"
    if (( doit )) ; then
	accumulateddata+="$data"
	if (( ++blockitemcount >= blocksize )) ; then
	    "${OC}" apply -f - <<< "$accumulateddata" | timestamp
	    accumulateddata=
	    blockitemcount=0
	    (( !sleeptime )) || sleep "$sleeptime"
	fi
    else
	echo "$data"
    fi
}

function createNamespace() {
    local namespace=$1
    createObject <<EOF
apiVersion: v1
kind: Namespace
metadata:
  name: "${namespace}"
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
EOF
}

function createSecrets() {
    local namespace=$1
    local depsPerNamespace=${2:-1}
    local secrets=${3:-1}
    local -i i
    local -i j
    (( secrets )) || return;
    for i in $(seq $firstDeployment $((depsPerNamespace + firstDeployment - 1))) ; do
	for j in $(seq 0 $((secrets - 1))) ; do
	    createObject <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: "secret-${namespace}-${i}-${j}"
  namespace: "$namespace"
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
data:
  key1: "$(base64 <<< "${namespace}X${i}Y${j}Z1")"
  key2: "$(base64 <<< "${namespace}X${i}Y${j}Z2")"
type: Opaque
EOF
	done
    done
}

function createVolumeMountsYAML() {
    local namespace=$1
    local deployment=${2:-1}
    local secrets=${3:-1}
    (( secrets + ${#emptydirs[@]})) || return;
    echo "volumeMounts:"
    local -i i
    for i in $(seq 0 $((secrets - 1))) ; do
	local name="secret-${namespace}-${deployment}-$i"
	cat <<EOF
- name: $name
  mountPath: /etc/$name
  readOnly: true
EOF
    done
    if [[ -n "${emptydirs[*]}" ]] ; then
	local vol
	for vol in "${emptydirs[@]}" ; do
	    cat <<EOF
- name: ${vol##*/}
  mountPath: "$vol"
EOF
	done
    fi
}

function createVolumesYAML() {
    local namespace=$1
    local deployment=${2:-1}
    local secrets=${3:-1}
    (( secrets + ${#emptydirs[@]})) || return;
    local -i i
    echo "volumes:"
    for i in $(seq 0 $((secrets - 1))) ; do
	local name="secret-${namespace}-${deployment}-$i"
	cat<<EOF
- name: $name
  secret:
    secretName: $name
EOF
    done

    if [[ -n "${emptydirs[*]}" ]] ; then
	local vol
	for vol in "${emptydirs[@]}" ; do
	    cat <<EOF
- name: ${vol##*/}
  emptydDir: {}
EOF
	done
    fi
}

function createContainersClusterBusterYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i pods=$4
    local -i containers=$5
    local -i j
    for j in $(seq 0 $((containers - 1))) ; do
	cat <<EOF
- name: ${namespace}-$i-$j
  imagePullPolicy: IfNotPresent
  image: "busybox"
$(indent 2 createContainerResources)
  command:
  - /bin/sh
  args:
  - -c
  - |
    trap 'exit 0' TERM
    (while : ; do date ; sleep 60 ; done)&
    sleep $podsleeptime
    exit 1
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
    done
}

function createContainersPausePodYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    # shellcheck disable=SC2034
    local -i pods=$4
    local -i containers=$5
    local -i j
    for j in $(seq 0 $((containers - 1))) ; do
	cat <<EOF
- name: ${namespace}-$i-$j
  imagePullPolicy: IfNotPresent
  image: "gcr.io/google_containers/pause-amd64:3.0"
$(indent 2 createContainerResources)
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
    done
}

function createClassicSpec() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i replicas=$4
    local -i containers=$5
cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
$(indent 2 createPodResources)
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createContainers${podType}YAML "$@")
$(indent 2 createVolumesYAML "$namespace" "$i" "$secretCount")
EOF
}

function createDeploymentClassicYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i replicas=$4
    local -i containers=$5
    if [[ $deploymentType = pod ]] ; then
	local -i j=0
	while (( j < replicas )) ; do
	    createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: ${namespace}-${i}-${j}
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-${i}
  labels:
    ${basename}: "true"
    name: ${namespace}-${i}
    app: ${namespace}-${i}
    ${basename}: "true"
    ${basename}-client: "true"
    clusterbusterbase: "true"
$(createClassicSpec "$@")
EOF
	    j=$((j+1))
	done
    else
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${namespace}-$i
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: $replicas
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-$i
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        ${basename}: "true"
        clusterbusterbase: "true"
        name: ${namespace}-$i
        app: ${namespace}-$i
$(indent 4 createClassicSpec "$@")
EOF
    fi
}

function syncPerl() {
    cat <<'EOF'
#!/usr/bin/perl
use Socket;
use POSIX;
use strict;
use Time::Piece;
use Time::HiRes qw(gettimeofday);
$SIG{TERM} = sub { POSIX::_exit(0); };
my ($listen_port, $expected_clients, $syncCount) = @ARGV;
if (! $syncCount) {
    $syncCount = 1;
}
sub timestamp($) {
    my ($str) = @_;
    my (@now) = gettimeofday();
    printf STDERR  "sync %s.%06d %s\n", gmtime($now[0])->strftime("%Y-%m-%dT%T"), $now[1], $str;
}
timestamp("Clusterbuster sync starting");
my $sockaddr = "S n a4 x8";
socket(SOCK, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
$SIG{TERM} = sub { close SOCK; POSIX::_exit(0); };
setsockopt(SOCK,SOL_SOCKET, SO_REUSEADDR, pack("l",1)) || die "setsockopt reuseaddr: $!\n";
setsockopt(SOCK,SOL_SOCKET, SO_KEEPALIVE, pack("l",1)) || die "setsockopt keepalive: $!\n";
bind(SOCK, pack($sockaddr, AF_INET, $listen_port, "\0\0\0\0")) || die "bind: $!\n";
listen(SOCK, 5) || die "listen: $!";
my $mysockaddr = getsockname(SOCK);
my ($junk, $port, $addr) = unpack($sockaddr, $mysockaddr);
die "can't get port $port: $!\n" if ($port ne $listen_port);
timestamp("Listening on port $listen_port");
my (@clients);

while ($syncCount < 0 || $syncCount-- > 0) {
    my $child = fork();
    if ($child == 0) {
	print STDERR "Expect $expected_clients clients\n";
	while ($expected_clients > 0) {
	    my ($client);
	    accept($client, SOCK) || next;
	    my $peeraddr = getpeername($client);
	    my ($port, $addr) = sockaddr_in($peeraddr);
	    my $peerhost = gethostbyaddr($addr, AF_INET);
	    my $peeraddr = inet_ntoa($addr);
	    my ($tbuf) = "NULL";
	    if (sysread($client, $tbuf, 1024) <= 0) {
		timestamp("Read token from $peerhost failed: $!");
	    }
	    timestamp("Accepted connection from $peerhost ($peeraddr) on $port, token $tbuf");
	    push @clients, $client;
	    $expected_clients--;
	}
	timestamp("Waiting 1 second to sync:");
	sleep(1);
	timestamp("Done!");
        POSIX::_exit(0);
    } elsif ($child < 1) {
        timestamp("Fork failed: $!");
	POSIX::_exit(1);
    } else {
        wait();
    }
}
POSIX::_exit(0);
EOF
}

function createSyncYAML() {
    local syncCount=${1:-1}
    local namespace=$2
    local expectedClients=$3
    cat <<EOF
- name: ${namespace}-sync
  imagePullPolicy: IfNotPresent
  image: perl
  ports:
  - containerPort: $port
  command:
  - perl
  args:
  - -e
  - |
$(indent 4 syncPerl)
  - "$syncPort"
  - "$expectedClients"
  - "$syncCount"
EOF
}

function createSyncSpec() {
    cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createSyncYAML "$@")
EOF
}


function createSyncDeployment() {
    local syncCount=$1; shift
    if [[ $deploymentType = pod ]] ; then
	createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: ${namespace}-sync
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-sync
  labels:
    name: ${namespace}-sync
    app: ${namespace}-sync
    k8s-app: ${namespace}-sync
    ${basename}: "true"
    ${basename}-sync: "true"
    clusterbusterbase: "true"
$(createSyncSpec "$syncCount" "$@")
  restartPolicy: Never
EOF
    else
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${namespace}-sync
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: 1
  strategy:
    type: RollingUpdate
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-sync
  template:
    metadata:
      labels:
        name: ${namespace}-sync
        app: ${namespace}-sync
        k8s-app: ${namespace}-sync
        ${basename}: "true"
        ${basename}-sync: "true"
        clusterbusterbase: "true"
$(indent 4 createSyncSpec -1 "$@")
EOF
    fi
}

function sysbenchPerl() {
    cat <<'EOF'
#!/usr/bin/perl

use Socket;
use POSIX;
use strict;
use Time::Piece;
use Time::HiRes qw(gettimeofday);
our ($namespace, $pod, $container, $basetime, $crtime, $poddelay, $processes, $rundir, $runtime, $exit_at_end, $synchost, $syncport, $loghost, $logport, $sysbench_generic_args, $sysbench_cmd, $sysbench_fileio_args, $sysbench_modes) = @ARGV;
$SIG{TERM} = sub { kill 'KILL', -1; POSIX::_exit(0); };

sub cputime() {
    my (@times) = times();
    return $times[0] + $times[1] + $times[2] + $times[3];
}

sub ts() {
    my (@now) = gettimeofday();
    return sprintf("%s.%06d", gmtime($now[0])->strftime("%Y-%m-%dT%T"), $now[1]);
}
sub timestamp($) {
    my ($str) = @_;
    printf STDERR "%7d %s %s\n", $$$$, ts(), $str;
}

sub xtime() {
    my (@now) = gettimeofday();
    return $now[0] + ($now[1] / 1000000.0);
}
sub connect_to($$$$) {
    my ($addr, $port) = @_;
    my ($connected) = 0;
    my ($ghbn_time, $stime);
    my ($fname,$faliases,$ftype,$flen,$faddr);
    my ($sock);
    do {
        ($fname,$faliases,$ftype,$flen,$faddr) = gethostbyname($addr);
        my $sockaddr = "S n a4 x8";
        if (length($faddr) < 4) {
            print STDERR "Malformed address, waiting for addr for $addr\n";
            sleep(1);
        } else {
            my $straddr = inet_ntoa($faddr);
            timestamp("Connecting to $addr:$port ($fname, $ftype)");
            $ghbn_time = xtime();
            my $sockmeta = pack($sockaddr, AF_INET, $port, $faddr);
            socket($sock, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "can't make socket: $!";
            $stime = xtime();
            if (connect($sock, $sockmeta)) {
                $connected = 1;
                timestamp("Connected to $addr:$port ($fname, $ftype), waiting for sync");
            } else {
                timestamp("Could not connect to $addr on port $port: $!");
                close $sock;
                sleep(1);
            }
        }
    } while (! $connected);
    return ($sock, $ghbn_time, $stime);
}

sub do_sync($$$$;$) {
    my ($addr, $port, $token) = @_;
    if (not $addr) { return; }
    if (not $token) {
        $token = sprintf('%d', rand() * 999999999);
    }
    timestamp("Waiting for sync on $addr:$port");
    my ($sync_conn, $i1, $i2) = connect_to($addr, $port);
    my ($sbuf);
    timestamp("Writing token $token to sync");
    my ($answer) = syswrite($sync_conn, $token, length $token);
    if ($answer != length $token) {
	timestamp("Write token failed: $!");
	exit(1);
    }
    $answer = sysread($sync_conn, $sbuf, 1024);
    my ($str) = sprintf("Got sync (%s, %d, %s)!", $answer, length $sbuf, $!);
    timestamp($str);
}

sub runit() {
    my ($files) = 0;
    my ($totalbytes) = 0;
    my ($seconds) = 0;
    my ($rate) = 0;
    my ($readops) = 0;
    my ($writeops) = 0;
    my ($fsyncops) = 0;
    my ($readrate) = 0;
    my ($writerate) = 0;
    my ($et) = 0;
    my ($min_lat) = 0;
    my ($avg_lat) = 0;
    my ($max_lat) = 0;
    my ($p95_lat) = 0;
    my (@known_sysbench_fileio_modes) = qw(seqwr seqrewr seqrd rndrd rndwr rndrw);
    my ($iterations) = 0;
    my ($loops_per_iteration) = 10000;
    if ($sysbench_modes) {
        @known_sysbench_fileio_modes = split(/ +/, $sysbench_modes);
    }
    timestamp(join("|", @known_sysbench_fileio_modes));
    my ($mode) = $known_sysbench_fileio_modes[int(rand() * ($#known_sysbench_fileio_modes + 1.0))];
    if (! -d $rundir) {
        timestamp("No run directory $rundir!");
	exit(1);
    }
    $rundir .= "/$$$$";
    if (! mkdir($rundir)) {
        timestamp("Cannot create run directory $rundir: $!");
	exit(1);
    }
    if (! chdir($rundir)) {
        timestamp("Cannot cd $rundir: $!");
	exit(1);
    }
    do_sync($synchost, $syncport);
    timestamp("Preparing...");
    open(PREPARE, "-|", "sysbench --time=$runtime $sysbench_generic_args $sysbench_cmd prepare --file-test-mode=$mode $sysbench_fileio_args") || die "Can't run sysbench: $!\n";
    while (<PREPARE>) {
	if ($_ =~ /^([[:digit:]]+) bytes written in ([[:digit:].]+) seconds/) {
	    $totalbytes = $1;
	    $seconds = $2;
	    $rate = $1 / $2;
	} elsif ($_ =~ /^Creating file /) {
	    $files++;
	    next;
	}
        if ($ENV{"VERBOSE"} > 0) {
       	    chomp;
	    timestamp($_);
	}
    }
    close PREPARE;

    my ($basecpu) = cputime();
    my ($prevcpu) = $basecpu;
    my ($firsttime) = 1;
    my ($avgcpu) = 0;
    my ($weight) = .25;
    my ($icputime);
    my ($interval) = 5;
    my $start_time;
    my ($dstime) = xtime();

    my $delaytime = $basetime + $poddelay - $dstime;
    do_sync($synchost, $syncport);
    my ($stime1) = xtime();
    my ($stime) = $stime1;
    my ($prevtime) = $stime;
    my ($scputime) = cputime();
    timestamp("Running...");
    open(RUN, "-|", "sysbench --time=$runtime $sysbench_generic_args $sysbench_cmd run --file-test-mode=$mode $sysbench_fileio_args") || die "Can't run sysbench: $!\n";
    while (<RUN>) {
	if ($_ =~ m;^[[:space:]]*reads/s:[[:space:]]*([[:digit:].]+);) {
	    $readops = $1;
	} elsif ($_ =~ m;^[[:space:]]*writes/s:[[:space:]]*([[:digit:].]+);) {
	    $writeops = $1;
	} elsif ($_ =~ m;^[[:space:]]*fsyncs/s:[[:space:]]*([[:digit:].]+);) {
	    $fsyncops = $1;
	} elsif ($_ =~ m;^[[:space:]]*read, MiB/s:[[:space:]]*([[:digit:].]+);) {
	    $readrate = $1 * 1048576.0;
	} elsif ($_ =~ m;^[[:space:]]*written, MiB/s:[[:space:]]*([[:digit:].]+);) {
	    $writerate = $1 * 1048576.0;
	} elsif ($_ =~ m;^[[:space:]]*total time:[[:space:]]*([[:digit:].]+)s;) {
	    $et = $1;
	} elsif ($_ =~ m;^[[:space:]]*min:[[:space:]]*([[:digit:].]+);) {
	    $min_lat = $1 / 1000.0;
	} elsif ($_ =~ m;^[[:space:]]*avg:[[:space:]]*([[:digit:].]+);) {
	    $avg_lat = $1 / 1000.0;
	} elsif ($_ =~ m;^[[:space:]]*max:[[:space:]]*([[:digit:].]+);) {
	    $max_lat = $1 / 1000.0;
	} elsif ($_ =~ m;^[[:space:]]*95th percentile:[[:space:]]*([[:digit:].]+);) {
	    $p95_lat = $1 / 1000.0;
	}
        if ($ENV{"VERBOSE"} > 0) {
       	    chomp;
	    timestamp($_);
	}
    }
    close RUN;
    open(CLEANUP, "-|", "sysbench --time=$runtime $sysbench_generic_args $sysbench_cmd cleanup --file-test-mode=$mode $sysbench_fileio_args") || die "Can't run sysbench: $!\n";
    while (<CLEANUP>) {
        if ($ENV{"VERBOSE"} > 0) {
       	    chomp;
	    timestamp($_);
	}
    }
    close CLEANUP;
    my ($answer) = sprintf("STATS %d %.3f %.3f %.3f %d %d %d %d %d %.03f %.06f %.06f %.06f %.06f",
	    $$$$, $crtime - $basetime, $dstime - $basetime, $stime1 - $basetime,
	    $readops, $writeops, $fsyncops, $readrate, $writerate, $et, $min_lat, $avg_lat, $max_lat, $p95_lat);
    print STDERR "$answer\n";
    do_sync($synchost, $syncport, $answer);
    do_sync($loghost, $logport, "-n $namespace $pod -c $container terminated 0 0 0 $answer");
}
$SIG{CHLD} = 'IGNORE';
if ($processes > 1) {
    for (my $i = 0; $i < $processes; $i++) {
        if ((my $child = fork()) == 0) {
            runit();
            exit(0);
        }
    }
} else {
    runit();
}
if ($exit_at_end) {
    timestamp("About to exit");
    while (wait() > 0) {}
    timestamp("Done waiting");
    print STDERR "FINIS\n";
    POSIX::_exit(0);
} else {
    timestamp("Waiting forever");
    pause()
}
EOF
}

function createContainersSysbenchYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    local log_host=${6:-}
    local -i log_port=${7:-0}
    local -i j
    local syncService=
    local syncPortNum=
    if (( syncStart )) ; then
	syncService="service-${namespace}-sync"
	syncPortNum=$syncPort
    fi
    # For prototype, just use /tmp.
    local workdir=${emptydirs[0]:-/tmp}
    for j in $(seq 0 $((containers - 1))) ; do
	cat <<EOF
- name: "c${j}"
  imagePullPolicy: IfNotPresent
  image: "severalnines/sysbench"
$(indent 2 createContainerResources)
  env:
  - name: VERBOSE
    value: "$verbose"
  command:
  - perl
  args:
  - -e
  - |
$(indent 4 sysbenchPerl)
  - "$namespace"
  - "${namespace}-sysbench-${i}"
  - "c${j}"
  - "$basetime"
  - "$(ts)"
  - "$poddelay"
  - "$processes"
  - "$workdir"
  - "$xferTime"
  - "$exitAtEnd"
  - "$syncService"
  - "$syncPortNum"
  - "$log_host"
  - "$log_port"
  - "${sysbench_generic_options[*]}"
  - "fileio"
  - "${sysbench_fileio_options[*]}"
  - "${!sysbench_fileio_tests[*]}"
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
    done
}

function createSysbenchSpec() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
  restartPolicy: Always
$(indent 2 createPodResources)
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createContainersSysbenchYAML "$@")
$(indent 2 createVolumesYAML "$namespace" "$i" "$secretCount")
EOF
}

function createSysbenchDeployment() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    if [[ $deploymentType = pod ]] ; then
	createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: ${namespace}-sysbench-${i}
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-sysbench-${i}
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
    name: ${namespace}-sysbench-${i}
    app: ${namespace}-sysbench-${i}
    ${basename}-sysbench: "true"
    ${basename}-client: "true"
$(createSysbenchSpec "$@")
  restartPolicy: Never
EOF
    else
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${namespace}-sysbench-$i
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: 1
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-sysbench-$i
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        ${basename}: "true"
        clusterbusterbase: "true"
        name: ${namespace}-sysbench-$i
        app: ${namespace}-sysbench-$i
        ${basename}-sysbench: "true"
        ${basename}-client: "true"
$(indent 4 createSysbenchSpec "$@")
EOF
    fi
}

function cpuSoakerPerl() {
    cat <<'EOF'
#!/usr/bin/perl

use Socket;
use POSIX;
use strict;
use Time::Piece;
use Time::HiRes qw(gettimeofday);
our ($namespace, $pod, $container, $basetime, $crtime, $poddelay, $processes, $runtime, $exit_at_end, $synchost, $syncport, $loghost, $logport) = @ARGV;
$SIG{TERM} = sub { kill 'KILL', -1; POSIX::_exit(0); };

sub cputime() {
    my (@times) = times();
    return $times[0] + $times[1] + $times[2] + $times[3];
}

sub ts() {
    my (@now) = gettimeofday();
    return sprintf("%s.%06d", gmtime($now[0])->strftime("%Y-%m-%dT%T"), $now[1]);
}
sub timestamp($) {
    my ($str) = @_;
    printf STDERR "%7d %s %s\n", $$$$, ts(), $str;
}

sub xtime() {
    my (@now) = gettimeofday();
    return $now[0] + ($now[1] / 1000000.0);
}
sub connect_to($$$$) {
    my ($addr, $port) = @_;
    my ($connected) = 0;
    my ($ghbn_time, $stime);
    my ($fname,$faliases,$ftype,$flen,$faddr);
    my ($sock);
    do {
        ($fname,$faliases,$ftype,$flen,$faddr) = gethostbyname($addr);
        my $sockaddr = "S n a4 x8";
        if (length($faddr) < 4) {
            print STDERR "Malformed address, waiting for addr for $addr\n";
            sleep(1);
        } else {
            my $straddr = inet_ntoa($faddr);
            timestamp("Connecting to $addr:$port ($fname, $ftype)");
            $ghbn_time = xtime();
            my $sockmeta = pack($sockaddr, AF_INET, $port, $faddr);
            socket($sock, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "can't make socket: $!";
            $stime = xtime();
            if (connect($sock, $sockmeta)) {
                $connected = 1;
                timestamp("Connected to $addr:$port ($fname, $ftype), waiting for sync");
            } else {
                timestamp("Could not connect to $addr on port $port: $!");
                close $sock;
                sleep(1);
            }
        }
    } while (! $connected);
    return ($sock, $ghbn_time, $stime);
}

sub do_sync($$$$;$) {
    my ($addr, $port, $token) = @_;
    my $otoken = $token;
    if (not $addr) { return; }
    if (not $token) {
        $token = sprintf('%d', rand() * 999999999);
    }
    timestamp("Waiting for sync on $addr:$port");
    my ($sync_conn, $i1, $i2) = connect_to($addr, $port);
    my ($sbuf);
    timestamp("Writing token $token to sync");
    my ($answer) = syswrite($sync_conn, $token, length $token);
    if ($answer != length $token) {
	timestamp("Write token failed: $!");
	exit(1);
    }
    $answer = sysread($sync_conn, $sbuf, 1024);
    my ($str) = sprintf("Got sync (%s, %d, %s)!", $answer, length $sbuf, $!);
    timestamp($str);
}

sub runit() {
    my ($iterations) = 0;
    my ($loops_per_iteration) = 10000;
    my ($basecpu) = cputime();
    my ($prevcpu) = $basecpu;
    my ($firsttime) = 1;
    my ($avgcpu) = 0;
    my ($weight) = .25;
    my ($icputime);
    my ($interval) = 5;
    my $start_time;
    my ($dstime) = xtime();

    my $delaytime = $basetime + $poddelay - $dstime;
    do_sync($synchost, $syncport);
    my ($stime1) = xtime();
    my ($stime) = $stime1;
    my ($prevtime) = $stime;
    my ($scputime) = cputime();
    while ($runtime < 0 || xtime() - $stime1 < $runtime) {
        my ($a) = 1;
        for (my $i = 0; $i < $loops_per_iteration; $i++) {
            $a = $a + $a;
        }
        $iterations += $loops_per_iteration;
        if ($ENV{"VERBOSE"} > 0) {
	    my ($ntime) = xtime();
	    if ($ntime - $prevtime >= $interval) {
		my (@times) = times();
		my ($user, $system, $cuser, $csystem) = times();
		my ($etime) = $ntime - $stime;
		my ($cpu) = cputime();
		my ($cputime) = $cpu - $basecpu;
		my ($icputime) = $cpu - $prevcpu;
		if ($firsttime) {
		    $avgcpu = $cputime;
		    $firsttime = 0;
		} else {
		    $avgcpu = ($icputime * $weight) + ($avgcpu * (1.0 - $weight));
		}
		printf STDERR "pid %d elapsed %.03f cpu %0.3f util %7.3f avg %7.3f mov %7.3f iter %d ips %d\n", $$$$, $etime, $cputime, 100.0 * $cputime / $etime, 100.0 * $icputime / ($ntime - $prevtime), 100 * $avgcpu / ($ntime - $prevtime), $iterations, $iterations / $etime;
		$prevtime = $ntime;
		$prevcpu = $cpu;
            }
        }
    }
    my ($etime) = xtime();
    my ($eltime) = $etime - $stime1;
    my ($cputime) = cputime() - $scputime;
    my ($answer) = sprintf("STATS %d %.3f %.3f %.3f %.3f %.3f %.3f %7.3f %d %d",
        $$$$, $crtime - $basetime, $dstime - $basetime, $stime1 - $basetime,
        $eltime, $etime - $basetime, $cputime, 100.0 * $cputime / $eltime, $iterations,
        $iterations / ($etime - $stime1));
    print STDERR "$answer\n";
    do_sync($synchost, $syncport, $answer);
    do_sync($loghost, $logport, "-n $namespace $pod -c $container terminated 0 0 0 $answer");
}
$SIG{CHLD} = 'IGNORE';
if ($processes > 1) {
    for (my $i = 0; $i < $processes; $i++) {
        if ((my $child = fork()) == 0) {
            runit();
            exit(0);
        }
    }
} else {
    runit();
}
print STDERR "FINIS\n";
if ($exit_at_end) {
    timestamp("About to exit");
    while (wait() > 0) {}
    timestamp("Done waiting");
    POSIX::_exit(0);
} else {
    timestamp("Waiting forever");
    pause()
}
EOF
}

function createContainersCPUSoakerYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    local log_host=${6:-}
    local -i log_port=${7:-0}
    local -i j
    local syncService=
    local syncPortNum=
    if (( syncStart )) ; then
	syncService="service-${namespace}-sync"
	syncPortNum=$syncPort
    fi
    for j in $(seq 0 $((containers - 1))) ; do
	cat <<EOF
- name: "c$j"
  imagePullPolicy: IfNotPresent
  image: perl
$(indent 2 createContainerResources)
  env:
  - name: VERBOSE
    value: "$verbose"
  command:
  - perl
  args:
  - -e
  - |
$(indent 4 cpuSoakerPerl)
  - "$namespace"
  - "${namespace}-soaker-${i}"
  - "c$j"
  - "$basetime"
  - "$(ts)"
  - "$poddelay"
  - "$processes"
  - "$xferTime"
  - "$exitAtEnd"
  - "$syncService"
  - "$syncPortNum"
  - "$log_host"
  - "$log_port"
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
    done
}

function createSoakerSpec() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
  restartPolicy: Always
$(indent 2 createPodResources)
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createContainersCPUSoakerYAML "$@")
$(indent 2 createVolumesYAML "$namespace" "$i" "$secretCount")
EOF
}

function createSoakerDeployment() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local -i processes=$4
    local -i containers=$5
    if [[ $deploymentType = pod ]] ; then
	createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: ${namespace}-soaker-${i}
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-soaker-${i}
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
    name: ${namespace}-soaker-${i}
    app: ${namespace}-soaker-${i}
    ${basename}-soaker: "true"
    ${basename}-client: "true"
$(createSoakerSpec "$@")
  restartPolicy: Never
EOF
    else
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${namespace}-soaker-$i
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: 1
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-soaker-$i
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        ${basename}: "true"
        clusterbusterbase: "true"
        name: ${namespace}-soaker-$i
        app: ${namespace}-soaker-$i
        ${basename}-soaker: "true"
        ${basename}-client: "true"
$(indent 4 createSoakerSpec "$@")
EOF
    fi
}

function createService() {
    local namespace=$1
    local deployment=$2
    local portnum=$3
    createObject <<EOF
apiVersion: v1
kind: Service
metadata:
  name: "service-${deployment}"
  namespace: "$namespace"
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  ports:
  - port: $portnum
    targetPort: $portnum
  selector:
    name: "${deployment}"
EOF
}

function createRoute() {
    local namespace=$1
    local deployment=$2
    createObject <<EOF
apiVersion: v1
kind: Route
metadata:
  name: "route-${deployment}"
  namespace: "$namespace"
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  host: "service-${deployment}"
  to:
    kind: Service
    name: "service-${deployment}"
  selector:
    name: "${deployment}"
EOF
}

function serverContainerPerl() {
    cat <<'EOF'
#!/usr/bin/perl
use Socket;
use POSIX;
use strict;
use Time::Piece;
use Time::HiRes qw(gettimeofday);
$SIG{TERM} = sub { POSIX::_exit(0); };
my ($basetime, $listen_port, $container, $msgSize, $ts, $expected_clients) = @ARGV;
sub timestamp($) {
    my ($str) = @_;
    my (@now) = gettimeofday();
    printf STDERR  "$container %s.%06d %s\n", gmtime($now[0])->strftime("%Y-%m-%dT%T"), $now[1], $str;
}
timestamp("Clusterbuster server starting");
my $sockaddr = "S n a4 x8";
socket(SOCK, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "socket: $!";
$SIG{TERM} = sub { close SOCK; kill 'KILL', -1; POSIX::_exit(0); };
setsockopt(SOCK,SOL_SOCKET, SO_REUSEADDR, pack("l",1)) || die "setsockopt reuseaddr: $!\n";
setsockopt(SOCK,SOL_SOCKET, SO_KEEPALIVE, pack("l",1)) || die "setsockopt keepalive: $!\n";
bind(SOCK, pack($sockaddr, AF_INET, $listen_port, "\0\0\0\0")) || die "bind: $!\n";
listen(SOCK, 5) || die "listen: $!";
my $mysockaddr = getsockname(SOCK);
my ($junk, $port, $addr) = unpack($sockaddr, $mysockaddr);
die "can't get port $port: $!\n" if ($port ne $listen_port);
timestamp("Listening on port $listen_port");
$SIG{CHLD} = 'IGNORE';

print STDERR "Expect $expected_clients clients\n";
while ($expected_clients != 0) {
    accept(CLIENT, SOCK) || next;
    if ((my $child = fork()) == 0) {
	close(SOCK);
	$SIG{TERM} = sub { close CLIENT; POSIX::_exit(0); };
	my $peeraddr = getpeername(CLIENT);
	my ($port, $addr) = sockaddr_in($peeraddr);
	my $peerhost = gethostbyaddr($addr, AF_INET);
	my $peeraddr = inet_ntoa($addr);
	timestamp("Accepted connection from $peerhost ($peeraddr) on $port!");
	my ($consec_empty) = 0;
	my $buffer;
	my $nread;
	my $ntotal = 0;
	my $nwrite;
	while (1) {
	    while ($ntotal < $msgSize && ($nread = sysread(CLIENT, $buffer, $msgSize, $ntotal)) > 0) {
		$ntotal += $nread;
		$consec_empty=0;
	    }
	    if ($nread < 0) {
		die "Write failed: $!\n";
	    }
	    if ($ntotal == 0) {
	        if ($consec_empty > 1) {
		    timestamp("Exiting $port");
		    exit(0);
		}
	        $consec_empty++;
	    }
	    while ($ntotal > 0 && ($nwrite = syswrite(CLIENT, $buffer, $ntotal)) > 0) {
		$ntotal -= $nwrite;
	    }
	    if ($nwrite < 0) {
		die "Write failed: $!\n";
	    }
	}
    } else {
	close(CLIENT);
	$expected_clients--;
    }
}
timestamp("Waiting for all clients to exit:");
while ((my $pid = wait()) >= 0) {
    timestamp("   $pid");
}
timestamp("Done!");
EOF
}

function createServerContainerYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local msgSize=$4
    local expectedClients=$5
    cat <<EOF
- name: server
  image: perl
  imagePullPolicy: IfNotPresent
$(indent 2 createContainerResources)
  ports:
  - containerPort: $port
  command:
  - perl
  args:
  - -e
  - |
$(indent 4 serverContainerPerl)
  - "$basetime"
  - "$port"
  - "${namespace}-${i}-server"
  - "$msgSize"
  - "$(ts)"
  - "$expectedClients"
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
}

function createServerSpec() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local msgSize=$4
    local expectedClients=$5
    cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
$(indent 2 createPodResources)
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createServerContainerYAML "$@")
$(indent 2 createVolumesYAML "$namespace" "$i" "$secretCount")
EOF
}

function createServerDeployment() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local msgSize=$4
    local expectedClients=$5
    if [[ $deploymentType = pod ]] ; then
	createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: ${namespace}-${i}-server
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-${i}
  labels:
    name: ${namespace}-${i}
    app: ${namespace}-${i}
    k8s-app: ${namespace}-${i}-server
    ${basename}: "true"
    ${basename}-server: "true"
    clusterbusterbase: "true"
$(createServerSpec "$@")
  restartPolicy: Never
EOF
    else
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: ${namespace}-${i}-server
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: 1
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-$i
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        name: ${namespace}-${i}
        app: ${namespace}-${i}
        k8s-app: ${namespace}-${i}-server
        ${basename}: "true"
        ${basename}-server: "true"
        clusterbusterbase: "true"
$(indent 4 createServerSpec "$@")
EOF
    fi
}

function clientContainerPerl() {
    cat <<'EOF'
#!/usr/bin/perl

use Socket;
use POSIX;
use strict;
use Time::HiRes qw(gettimeofday usleep);
use Time::Piece;
$SIG{TERM} = sub { kill 'KILL', -1; POSIX::_exit(0); };
my ($namespace, $pod, $container, $basetime, $poddelay, $connect_port, $container, $srvhost, $data_rate, $bytes, $bytesMax, $msgSize, $xfertime, $xfertimeMax, $crtime, $exit_at_end, $synchost, $syncport, $namespace, $log_host, $log_port, $podname) = @ARGV;
my ($etime, $data_sent, $detime, $stime, $end_time, $dstime, $mean, $stdev, $user, $sys, $cuser, $csys, $elapsed);
my $start_time;
my $ghbn_time;
my $pass = 0;
my $e0 = 0;
my $ex = 0;
my $ex2 = 0;
my ($cfail) = 0;
my ($refused) = 0;
my $time_overhead = 0;
sub ts() {
    my (@now) = gettimeofday();
    return sprintf("%s.%06d", gmtime($now[0])->strftime("%Y-%m-%dT%T"), $now[1]);
}
sub timestamp($) {
    my ($str) = @_;
    printf STDERR "%s %s %s\n", $container, ts(), $str;
}
sub xtime() {
    my (@now) = gettimeofday();
    return $now[0] + ($now[1] / 1000000.0);
}
sub calibrate_time() {
    for (my $i = 0; $i < 1000; $i++) {
        my ($start) = xtime();
	my ($end) = xtime();
	$time_overhead += $end - $start;
    }
    $time_overhead /= 1000;
}
sub stats() {
    return sprintf("STATS %.3f %.3f %.3f %.3f %.3f %.3f %.6f %.3f %.3f %d %.3f %.3f %.6f %.6f %.6f",
	   $crtime - $basetime, $start_time - $basetime, $ghbn_time - $basetime, $etime - $basetime,
	   $dstime - $basetime, $end_time - $basetime, $elapsed, $user, $sys,
	   $data_sent, $detime, $data_sent / $detime / 1000000.0, $mean, $stdev, $time_overhead);
}
sub connect_to($$$$) {
    my ($addr, $port) = @_;
    my ($connected) = 0;
    my ($ghbn_time, $stime);
    my ($fname,$faliases,$ftype,$flen,$faddr);
    my ($sock);
    do {
        ($fname,$faliases,$ftype,$flen,$faddr) = gethostbyname($addr);
        my $sockaddr = "S n a4 x8";
        if (length($faddr) < 4) {
            print STDERR "Malformed address, waiting for addr for $addr\n";
	    $cfail++;
            sleep(1);
        } else {
            my $straddr = inet_ntoa($faddr);
            timestamp("Connecting to $addr:$port ($fname, $ftype)");
            $ghbn_time = xtime();
            my $sockmeta = pack($sockaddr, AF_INET, $port, $faddr);
            socket($sock, AF_INET, SOCK_STREAM, getprotobyname('tcp')) || die "can't make socket: $!";
            $stime = xtime();
            if (connect($sock, $sockmeta)) {
                $connected = 1;
            } else {
	        $cfail++;
		if ($! =~ /refused/) {
		    $refused++;
		}
                timestamp("Could not connect to $addr on port $port: $!");
                close $sock;
                sleep(1);
            }
        }
    } while (! $connected);
    return ($sock, $ghbn_time, $stime);
}
$SIG{TERM} = sub { POSIX::_exit(0); };
timestamp("Clusterbuster client starting");
$start_time = xtime();
my ($conn);
($conn, $ghbn_time, $stime) = connect_to($srvhost, $connect_port);
$SIG{TERM} = sub { close $conn; POSIX::_exit(0); };

sub do_sync($$$$;$) {
    my ($addr, $port, $token) = @_;
    if (not $addr) { return; }
    if (not $token) {
        $token = sprintf('%d', rand() * 999999999);
    }
    timestamp("Waiting for  on $addr:$port");
    my ($_conn, $i1, $i2) = connect_to($addr, $port);
    my ($sbuf);
    timestamp("Writing token $token to sync");
    my ($answer) = syswrite($_conn, $token, length $token);
    if ($answer != length $token) {
	timestamp("Write token failed: $!");
	exit(1);
    }
    $answer = sysread($_conn, $sbuf, 1024);
    my ($str) = sprintf("Got sync (%s, %d, %s)!", $answer, length $sbuf, $!);
    timestamp($str);
}
do_sync($synchost, $syncport);
$etime = xtime();
$elapsed = $etime - $stime;
my $peeraddr = getpeername($conn);
my ($port, $addr) = sockaddr_in($peeraddr);
my $peerhost = gethostbyaddr($addr, AF_INET);
$peeraddr = inet_ntoa($addr);
timestamp("Connected to $peerhost ($peeraddr) on port tcp:$port");
my $buffer = "";
vec($buffer, $msgSize - 1, 8) = "A";
my $nread;
my $bufsize = length($buffer);
my $starttime = xtime();
my $MBSec = $data_rate * 1;
($dstime) = xtime();

$data_sent = 0;
$mean = 0;
$stdev = 0;
if ($bytes != $bytesMax) {
    $bytes += int(rand($bytesMax - $bytes + 1));
}
if ($xfertime != $xfertimeMax) {
    $xfertime += int(rand($xfertimeMax - $xfertime + 1));
}
if ($MBSec != 0) {
    calibrate_time();
    my $delaytime = $basetime + $poddelay - $dstime;
    timestamp("Using $bufsize byte buffer");
    if ($delaytime > 0) {
	timestamp("Sleeping $delaytime seconds to synchronize");
	usleep($delaytime * 1000000);
    }
    $dstime = xtime();
    my ($tbuf, $rtt_start, $rtt_end, $rtt_elapsed);
    while (($bytes > 0 && $data_sent < $bytes) ||
	   ($xfertime > 0 && xtime() - $dstime < $xfertime)) {
	my $nwrite;
	my $nleft = $bufsize;
	while ($nleft > 0 && ($nwrite = syswrite($conn, $buffer, $nleft)) > 0) {
	    $nleft -= $nwrite;
	    $data_sent += $nwrite;
	}
	if ($nwrite == 0) {
	    exit 0;
	} elsif ($nwrite < 0) {
	    die "Write failed: $!\n";
	}
	$nleft = $bufsize;
	 $rtt_start = xtime();
	while ($nleft > 0 && ($nread = sysread($conn, $tbuf, $nleft)) > 0) {
	    $nleft -= $nread;
	}
        $rtt_end = xtime();
	$rtt_elapsed = $rtt_end - $rtt_start - $time_overhead;
	if ($pass == 0) {
	    $e0 = $rtt_elapsed - $;
	}
	my $en = $rtt_elapsed - $e0;
	$ex += $en;
	$ex2 += $en * $en;
	if ($nread < 0) {
	    die "Read failed: $!\n";
	}
	if ($ENV{"VERBOSE"} > 0) {
	    timestamp(sprintf("Write/Read %d %.6f", $bufsize, $rtt_elapsed));
	}
	my $curtime = xtime();
	$starttime += $bufsize / ($MBSec * 1000000);
	if ($curtime < $starttime && $MBSec > 0) {
	    if ($ENV{"VERBOSE"} > 0) {
		timestamp(sprintf("Sleeping %8.6f", $starttime - $curtime));
	    }
	    usleep(($starttime - $curtime) * 1000000);
	} else {
	    if ($ENV{"VERBOSE"} > 0 && $MBSec > 0) {
		timestamp("Not sleeping");
	    }
	}
	$pass++;
    }
}
if ($pass > 0) {
    $mean = ($ex / $pass) + $e0;
    if ($pass > 1) {
	$stdev = sqrt(($ex2 - ($ex * $ex / $pass)) / ($pass - 1));
    }
}
($user, $sys, $cuser, $csys) = times;
$end_time = xtime();
$detime = $end_time - $dstime;
if ($detime <= 0) {
    $detime = 0.00000001;
}

timestamp("Done");
my ($results) = stats();
print STDERR "$results\n";
print STDERR "FINIS\n";
if ($syncport) {
    do_sync($synchost, $syncport, "-n $namespace $pod -c $container $results");
}
if ($log_port > 0) {
    do_sync($log_host, $log_port, "-n $namespace $pod -c $container terminated $cfail $refused $pass $results");
}
if (! $exit_at_end) {
    pause();
}
exit 0;
EOF
}

function createClientContainerYAML() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local dataRate=$4
    local bytes=$5
    local msgSize=$6
    local containers=$7
    local log_host=$8
    local log_port=$9
    local podname=${10}
    local replica=${11}
    local -i j
    local syncService=
    local syncPortNum=
    if (( syncStart )) ; then
	syncService="service-${namespace}-sync"
	syncPortNum=$syncPort
    fi
    for j in $(seq 0 $((containers - 1))) ; do
	cat <<EOF
- name: c${j}
  image: "perl"
  imagePullPolicy: IfNotPresent
$(indent 2 createContainerResources)
  env:
  - name: VERBOSE
    value: "$verbose"
  command:
  - perl
  args:
  - -e
  - |
$(indent 4 clientContainerPerl)
  - "$namespace"
  - "${namespace}-${i}-client-$replica"
  - "$j"
  - "$basetime"
  - "$poddelay"
  - "$port"
  - "c${j}"
  - "service-${namespace}-$i"
  - "$dataRate"
  - "$bytes"
  - "$bytesMax"
  - "$msgSize"
  - "$xferTime"
  - "$xferTimeMax"
  - "$(ts)"
  - "$exitAtEnd"
  - "$syncService"
  - "$syncPortNum"
  - "$namespace"
  - "$log_host"
  - "$log_port"
  - "$podname"
$(indent 2 createVolumeMountsYAML "$namespace" "$i" "$secretCount")
EOF
    done
}

function createClientAffinityYAML() {
    (( affinity )) || return 0
    local server=$1
    local affinityType
    case "$affinity" in
	1) affinityType=Affinity     ;;
	*) affinityType=AntiAffinity ;;
    esac
    cat << EOF
affinity:
  pod$affinityType:
    requiredDuringSchedulingIgnoredDuringExecution:
    - labelSelector:
        matchExpressions:
        - key: k8s-app
          operator: In
          values:
          - "$server"
      topologyKey: kubernetes.io/hostname
EOF
}

function createClientSpec() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local replicas=$4
    local dataRate=$5
    local bytes=$6
    local msgSize=$7
    local containers=$8
    local log_host=${9:-}
    local -i log_port=${10:-0}
    local podname=${11}
    local replica=${12}
    cat <<EOF
spec:
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  terminationGracePeriodSeconds: 1
$(indent 2 createPodResources)
$(indent 2 createClientAffinityYAML "${namespace}-${i}-server")
$(indent 2 <<< "$runtimeClass")
  containers:
$(indent 2 createClientContainerYAML "$namespace" "$i" "$secretCount" "$dataRate" "$bytes" "$msgSize" "$containers" "$log_host" "$log_port" "$podname" "$replica")
$(indent 2 createVolumesYAML "$namespace" "$i" "$secretCount")
EOF
}

function createClientDeployment() {
    local namespace=$1
    local i=$2
    local secretCount=$3
    local replicas=$4
    local dataRate=$5
    local bytes=$6
    local msgSize=$7
    local containers=$8
    local log_host=${9:-}
    local -i log_port=${10:-0}
    if [[ $deploymentType = pod ]] ; then
	local -i j=0
	while (( j < replicas )) ; do
	    local podname="${namespace}-${i}-client-${j}"
	    createObject <<EOF
apiVersion: v1
kind: Pod
metadata:
  name: $podname
  namespace: $namespace
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  selector:
    matchLabels:
      app: ${namespace}-${i}-client
  labels:
    ${basename}: "true"
    name: ${namespace}-${i}-client
    app: ${namespace}-${i}-client
    ${basename}: "true"
    ${basename}-client: "true"
    clusterbusterbase: "true"
$(createClientSpec "$@" "$podname" "$j")
  restartPolicy: Never
EOF
	    j=$((j+1))
	done
    else
	local podname="${namespace}-${i}-client"
	createObject <<EOF
apiVersion: apps/v1
kind: Deployment
metadata:
  name: $podname
  namespace: $namespace
  labels:
    ${basename}: "true"
    clusterbusterbase: "true"
spec:
  replicas: $replicas
  nodeSelector:
    node-role.kubernetes.io/worker: ""
  restartPolicy: Never
  selector:
    matchLabels:
      app: ${namespace}-${i}-client
  strategy:
    type: RollingUpdate
  template:
    metadata:
      labels:
        name: ${namespace}-${i}-client
        app: ${namespace}-${i}-client
        ${basename}: "true"
        ${basename}-client: "true"
        clusterbusterbase: "true"
$(indent 4 createClientSpec "$@" "$podname" "x")
EOF
    fi
}

function createDeploymentSimple() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    local -i i
    for i in $(seq $firstDeployment $((count + firstDeployment - 1))) ; do
	createDeploymentClassicYAML "$namespace" "$i" "$secretCount"  "$replicas" "$containers"
    done
}

function createSyncSimple() {
    :
}

function calculateLogsRequiredSimple() {
    echo 0
}

function createDeploymentPausePod() {
    createDeploymentSimple "$@"
}

function createSyncPausePod() {
    :
}

function calculateLogsRequiredPausePod() {
    echo 0
}

function createDeploymentClusterBuster() {
    createDeploymentSimple "$@"
}

function createSyncClusterBuster() {
    :
}

function calculateLogsRequiredClusterBuster() {
    echo 0
}

function createDeploymentSysbench() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    local log_host=${6:-}
    local -i log_port=${7:-0}
    local -i i
    if (( syncStart )) ; then
	createService "$namespace" "${namespace}-sync" "$syncPort"
	createSyncDeployment 3 "$namespace" "$((containers * replicas * count))"
    fi
    for i in $(seq $firstDeployment $((count + firstDeployment - 1))) ; do
	createSysbenchDeployment "$namespace" "$i" "$secretCount"  "$replicas" "$containers" "$log_host" "$log_port"
    done
}

function createSyncSysbench() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    createService "$namespace" "${namespace}-sync" "$syncPort"
    createSyncDeployment 3 "$namespace" "$((containers * replicas * count))"
}

function calculateLogsRequiredSysbench() {
    local -i namespaces=$1
    local -i depsPerNamespace=${2:-1}
    local -i replicas=${3:-1}
    local -i containers=${4:-1}
    echo $((namespaces * containers * replicas * depsPerNamespace))
}

function createDeploymentCPUSoaker() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    local log_host=${6:-}
    local -i log_port=${7:-0}
    local -i i
    if (( syncStart )) ; then
	createService "$namespace" "${namespace}-sync" "$syncPort"
	createSyncDeployment 2 "$namespace" "$((containers * replicas * count))"
    fi
    for i in $(seq $firstDeployment $((count + firstDeployment - 1))) ; do
	createSoakerDeployment "$namespace" "$i" "$secretCount"  "$replicas" "$containers" "$log_host" "$log_port"
    done
}

function createSyncCPUSoaker() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    createService "$namespace" "${namespace}-sync" "$syncPort"
    createSyncDeployment 2 "$namespace" "$((containers * replicas * count))"
}

function calculateLogsRequiredCPUSoaker() {
    local -i namespaces=$1
    local -i depsPerNamespace=${2:-1}
    local -i replicas=${3:-1}
    local -i containers=${4:-1}
    echo $((namespaces * containers * replicas * depsPerNamespace))
}

function createDeploymentServer() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    local log_host=${6:-}
    local -i log_port=${7:-0}
    local -i i
    if (( syncStart )) ; then
	createService "$namespace" "${namespace}-sync" "$syncPort"
	createSyncDeployment 2 "$namespace" "$((containers * replicas * count))"
    fi

    for i in $(seq $firstDeployment $((count + firstDeployment - 1))) ; do
	createService "$namespace" "$namespace-$i" "$port"
	(( createRoutes )) && createRoute "$namespace" "$namespace-$i"
	if [[ $deploymentType = pod ]] ; then
	    createServerDeployment "$namespace" "$i" "$secretCount" "$msgSize" "$((containers * replicas))"
	else
	    createServerDeployment "$namespace" "$i" "$secretCount" "$msgSize" "-1"
	fi
	createClientDeployment "$namespace" "$i" "$secretCount" "$replicas" "$dataRate" "$bytes" "$msgSize" "$containers" "$log_host" "$log_port"
    done
}

function calculateLogsRequiredServer() {
    local -i namespaces=$1
    local -i depsPerNamespace=${2:-1}
    local -i replicas=${3:-1}
    local -i containers=${4:-1}
    echo $((namespaces * containers * replicas * depsPerNamespace))
}

function createSyncServer() {
    local namespace=$1
    local count=${2:-1}
    local secretCount=${3:-1}
    local replicas=${4:-1}
    local containers=${5:-1}
    createService "$namespace" "${namespace}-sync" "$syncPort"
    createSyncDeployment 2 "$namespace" "$((containers * replicas * count))"
}

function createDeployment() {
    "createDeployment${podType}" "$@"
}

function createSync() {
    if (( syncStart )) ; then
	"createSync${podType}" "$@"
    fi
}

function createObjects_n() {
    trap exit INT
    local objtype=$1; shift
    local parallel=$1; shift
    local blocksize=$1; shift
    local rotor=$1; shift
    local sleeptime=$1; shift
    while (( rotor < namespaces )) ; do
	local j=0
	while (( j < blocksize && j + rotor < namespaces )) ; do
	    "create${objtype}" "${namespaces_to_create[$((rotor + j))]}" "$@"
	    if (( sleeptime )) ; then sleep "$sleeptime"; fi
	    j=$((j+1))
	done
	rotor=$((rotor + (parallel * blocksize)))
    done
    finishCreation
}

function allocate_namespaces() {
    local -i ns_count=0
    local -i ns_idx=0
    if (( scale_ns )) ; then
        while read -r ns ; do
	    if [[ -n "$ns" ]] ; then
		namespaces_in_use[${ns#namespace/}]=1
	    fi
        done <<< "$("$OC" get ns -l "$basename" --no-headers -o name 2>/dev/null)"
    fi
    while (( ns_count++ < namespaces )) ; do
        while [[ -n "${namespaces_in_use[${basename}-$ns_idx]:-}" ]] ; do
            ns_idx=$((ns_idx+1))
        done
        namespaces_to_create+=("${basename}-$((ns_idx++))")
    done
}

function find_first_deployment() {
    if (( scale_deployments )) ; then
	local ns
	local deployment
	local stuff
	# shellcheck disable=SC2034
        while read -r ns deployment stuff ; do
	    if [[ -n "$deployment" ]] ; then
		deployment=${deployment#${ns}-}
		deployment=${deployment%-*}
		if (( deployment + 1 > firstDeployment )) ; then
		    firstDeployment=$((deployment + 1))
		fi
	    fi
        done <<< "$("$OC" get deployments -l "$basename" -A --no-headers 2>/dev/null)"
    fi
}

function createAllNamespaces() {
    local i
    trap 'kill -9 $(jobs -p); exit' INT
    for i in $(seq 0 "$((parallelNamespaces - 1))") ; do
	createObjects_n Namespace "$parallelNamespaces" "$blocksizeNamespaces" "$((i * blocksizeNamespaces))" "$sleepNamespaces" &
    done
    wait
}

function createAllSecrets() {
    local i
    trap 'kill -9 $(jobs -p); exit' INT
    for i in $(seq 0 "$((parallelSecrets - 1))") ; do
	createObjects_n Secrets "$parallelSecrets" "$blocksizeSecrets" "$((i * blocksizeSecrets))" "$sleepSecrets" "$depsPerNamespace" "$secrets" &
    done
    wait
}

function createAllDeployments() {
    local i
    trap 'kill -9 $(jobs -p); exit' INT
    for i in $(seq 0 "$((parallelDeployments - 1))") ; do
	export PARALLEL_ROTOR=$i
	createObjects_n Deployment "$parallelDeployments" "$blocksizeDeployments" "$((i * blocksizeDeployments))" "$sleepDeployments" "$depsPerNamespace" "$secrets" "$replicas" "$containers" "$log_host" "$log_port" &
    done
    wait
}

function createAllObjects() {
    local found_objtype
    local objtype=$1
    shift
    while read -r line ; do
	if [[ $line =~ ^[[:digit:]]{4}(-[[:digit:]]{2}){2}T([[:digit:]]{2}:){2}[[:digit:]]{2}\.[[:digit:]]{6}\ +([-a-z0-9]+)/([-a-z0-9]+)\ +created$ ]] ; then
	    if (( reportObjectCreation )) ; then
		echo "$line"
	    fi
	    totalObjectsCreated=$((totalObjectsCreated+1))
	    found_objtype=${BASH_REMATCH[3]}
	    if [[ -z ${objectsCreated[$found_objtype]:-} ]] ; then
		objectsCreated[$found_objtype]=1
	    else
		objectsCreated[$found_objtype]=$((${objectsCreated[$found_objtype]:-}+1))
	    fi
	elif [[ -n "$line" ]] ; then
	    echo "$line"
	fi
    done <<< "$("createAll${objtype}" "$@" 2>&1)"
}

function run_clusterbuster_1() {
    local -A expected_secrets=()
    local -i i
    local -i status=0
    allocate_namespaces
    find_first_deployment
    trap 'kill -INT $(jobs -p); exit' INT
    createAllObjects Namespaces

    # If previous secrets weren't all created, this will yield
    # the wrong result.
    if (( doit && wait_for_secrets )) ; then
	local -i i
	local -i j
	local -i k
	local ns
	local nsd
	for i in $(seq 0 $((namespaces - 1))) ; do
	    ns="secret/secret-${basename}-${namespaces_to_create[$i]##*-}"
	    for j in $(seq $firstDeployment $((depsPerNamespace + firstDeployment - 1))) ; do
		nsd="${ns}-$j"
		for k in $(seq 0 $((secrets - 1))) ; do
		    expected_secrets["${nsd}-$k"]=1
		done
	    done
	done
    fi

    createAllObjects Secrets

    local secname=""
    while (( ${#expected_secrets[@]} )) ; do
	while read -r secname ; do
	    [[ -n "${secname:-}" ]] && unset "expected_secrets[$secname]"
	done <<< "$("$OC" get secret -oname --no-headers -l "$basename" -A)"
	if (( ${#expected_secrets[@]} )) ; then
	    echo "Still waiting for ${#expected_secrets[@]} to be created."
	    sleep 10
	else
	    break
	fi
    done

    basetime=$(printf '%(%s)T' -1)

    local -i logs_expected=0
    local log_host=
    local -i log_port=0
    if ((report)) ; then
	logs_expected="$("calculateLogsRequired${podType}" "$namespaces" "$depsPerNamespace" "$replicas" "$containers")"
	if ((logs_expected)) ; then
	    log_host=$(hostname -f)
	    log_port=$(find_free_port)
	    (( log_port )) || fatal "Can't get port to listen for logs on!"
	    get_logs -p "$log_port" -c "$logs_expected" &
	fi
    fi
    createAllObjects Deployments

    if ((report && logs_expected && timeout > 0)) ; then
	local -i finis=0
	while (( timeout < 0 || timeout-- )) ; do
	    # This appears to be needed for the shell to actually reap
	    # any terminated jobs.  Without this, the test below always
	    # succeeds so the job doesn't exit.
	    jobs -l >/dev/null
	    if jobs -l |grep -q . ; then
		sleep 1
	    else
		finis=1
		break
	    fi
	done
	if (( ! finis )) ; then
	    exec 3>&2 2>/dev/null
	    kill %get_logs
	    exec 2>&3 3>&-
	    echo "Run did not terminate" 1>&2
	    status=1
	fi
    fi
    exec 3>&2 2>/dev/null
    wait
    exec 2>&3 3>&-

    local objtype
    while read -r objtype ; do
	[[ -n "${objtype:-}" ]] && printf "Created %${#totalObjectsCreated}d %s(s)\n" "${objectsCreated[$objtype]}" "$objtype"
    done <<< "$( (IFS=$'\n'; echo "${!objectsCreated[*]}") | sort)"
    echo "Created $totalObjectsCreated object(s) total"
    if ((cleanup)) ; then
	if (( reportObjectCreation )) ; then
	    oc delete ns -l "$basename"
	else
	    echo "Deleted  $(oc delete ns -l "$basename" |wc -l) namespace(s)"
	fi
    fi

    return $status
    # Create sync pods after actual deployments
#    if (( syncStart )) ; then
#	for i in $(seq 0 "$((parallelDeployments - 1))") ; do
#	    export PARALLEL_ROTOR=$i
#	    createObjects_n Sync "$parallelDeployments" "$blocksizeDeployments" "$((i * blocksizeDeployments))" "$sleepDeployments" "$depsPerNamespace" "$secrets" "$replicas" "$containers" &
#	done
#	wait
#    fi
}

(run_clusterbuster_1)