Enable node-density lite on windows workers (#462)

* PERFSCALE-1848 - Enable windows workers

* Revert tolerations variable. Add to pod template outright.

workloads/kube-burner/README.md

@@ -38,6 +38,7 @@ Workloads can be tweaked with the following environment variables:
 | **QPS**              | Queries/sec                     | 20      |
 | **BURST**            | Maximum number of simultaneous queries | 20      |
 | **POD_NODE_SELECTOR**| nodeSelector for pods created by the kube-burner workloads | {node-role.kubernetes.io/worker: } |
+| **WORKER_NODE_LABEL**| Label to select nodes used for the kube-burner workloads | node-role.kubernetes.io/worker |
 | **POD_WAIT**         | Wait for pods to be ready in each iteration | false |
 | **MAX_WAIT_TIMEOUT** | Kube-burner will time out when the pods deployed take more that this value to be ready | 1h |
 | **WAIT_FOR**         | Wait for the resources of this list to be ready | [] (empty means all of them) |
@@ -85,8 +86,9 @@ Each iteration creates the following objects:
 
 The `node-density` and `node-density-heavy` workloads support the following environment variables:
 
-- **NODE_COUNT**: Number of worker nodes to deploy the pods on. During the workload nodes will be labeled with `node-density=enabled`. Defaults to the number of worker nodes across the cluster (Nodes resulting of the expression `oc get node -o name --no-headers -l node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="",node-role.kubernetes.io/worker=`
+- **NODE_COUNT**: Number of worker nodes to deploy the pods on. During the workload nodes will be labeled with `node-density=enabled`. Defaults to the number of worker nodes across the cluster (Nodes resulting of the expression `oc get node -o name --no-headers -l node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="",${WORKER_NODE_LABEL}`
 - **PODS_PER_NODE**: Define the maximum number of pods to deploy on each labeled node. Defaults to 245
+- **NODE_POD_DENSITY_IMAGE**: Image to use as node-pod-density workload. Defaults to `gcr.io/google_containers/pause:3.1`.
 
 These workloads create different objects each:
 

workloads/kube-burner/common.sh

@@ -212,13 +212,13 @@ label_node_with_label() {
   colon_param=$(echo $1 | tr "=" ":" | sed 's/:/: /g')
   export POD_NODE_SELECTOR="{$colon_param}"
   if [[ -z $NODE_COUNT ]]; then
-    NODE_COUNT=$(oc get node -o name --no-headers -l node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="",node-role.kubernetes.io/worker= | wc -l )
+    NODE_COUNT=$(oc get node -o name --no-headers -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= | wc -l )
   fi
   if [[ ${NODE_COUNT} -le 0 ]]; then
     log "Node count <= 0: ${NODE_COUNT}"
     exit 1
   fi
-  WORKER_NODE_NAMES=$(oc get node -o custom-columns=name:.metadata.name --no-headers -l node-role.kubernetes.io/workload!="",node-role.kubernetes.io/infra!="",node-role.kubernetes.io/worker= | head -n ${NODE_COUNT})
+  WORKER_NODE_NAMES=$(oc get node -o custom-columns=name:.metadata.name --no-headers -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= | head -n ${NODE_COUNT})
   if [[ $(echo "${WORKER_NODE_NAMES}" | wc -l) -lt ${NODE_COUNT} ]]; then
     log "Not enough worker nodes to label"
     exit 1

workloads/kube-burner/env.sh

@@ -17,6 +17,7 @@ export BURST=${BURST:-20}
 export MAX_WAIT_TIMEOUT=${MAX_WAIT_TIMEOUT:-1h}
 export CLEANUP=${CLEANUP:-true}
 export POD_NODE_SELECTOR=${POD_NODE_SELECTOR:-'{node-role.kubernetes.io/worker: }'}
+export WORKER_NODE_LABEL=${WORKER_NODE_LABEL:-"node-role.kubernetes.io/worker"}
 export WAIT_WHEN_FINISHED=true
 export POD_WAIT=${POD_WAIT:-false}
 export WAIT_FOR=${WAIT_FOR:-[]}
@@ -32,6 +33,9 @@ export JOB_TIMEOUT=${JOB_TIMEOUT:-14400}
 export LOG_STREAMING=${LOG_STREAMING:-true}
 export METRICS_PROFILE=${METRICS_PROFILE}
 
+# kube-burner workload defaults
+export NODE_POD_DENSITY_IMAGE=${NODE_POD_DENSITY_IMAGE:-gcr.io/google_containers/pause:3.1}
+
 # Misc
 export CLEANUP_WHEN_FINISH=${CLEANUP_WHEN_FINISH:-false}
 export CLEANUP_TIMEOUT=${CLEANUP_TIMEOUT:-30m}

workloads/kube-burner/run.sh

@@ -16,7 +16,7 @@ case ${WORKLOAD} in
   node-density)
     WORKLOAD_TEMPLATE=workloads/node-pod-density/node-pod-density.yml
     METRICS_PROFILE=${METRICS_PROFILE:-metrics-profiles/metrics.yaml}
-    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l node-role.kubernetes.io/worker,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
+    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
     PODS_PER_NODE=${PODS_PER_NODE:-245}
     label="node-density=enabled"
     label_node_with_label $label
@@ -25,7 +25,7 @@ case ${WORKLOAD} in
   node-density-heavy)
     WORKLOAD_TEMPLATE=workloads/node-density-heavy/node-density-heavy.yml
     METRICS_PROFILE=${METRICS_PROFILE:-metrics-profiles/metrics.yaml}
-    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l node-role.kubernetes.io/worker,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
+    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
     PODS_PER_NODE=${PODS_PER_NODE:-245}
     label="node-density=enabled"
     label_node_with_label $label
@@ -34,7 +34,7 @@ case ${WORKLOAD} in
   node-density-cni)
     WORKLOAD_TEMPLATE=workloads/node-density-cni/node-density-cni.yml
     METRICS_PROFILE=${METRICS_PROFILE:-metrics-profiles/metrics.yaml}
-    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l node-role.kubernetes.io/worker,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
+    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
     PODS_PER_NODE=${PODS_PER_NODE:-245}
     label="node-density=enabled"
     label_node_with_label $label
@@ -43,7 +43,7 @@ case ${WORKLOAD} in
   node-density-cni-networkpolicy)
     WORKLOAD_TEMPLATE=workloads/node-density-cni-networkpolicy/node-density-cni-networkpolicy.yml
     METRICS_PROFILE=${METRICS_PROFILE:-metrics-profiles/metrics.yaml}
-    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l node-role.kubernetes.io/worker,node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
+    NODE_COUNT=${NODE_COUNT:-$(kubectl get node -l ${WORKER_NODE_LABEL},node-role.kubernetes.io/infra!=,node-role.kubernetes.io/workload!= -o name | wc -l)}
     PODS_PER_NODE=${PODS_PER_NODE:-245}
     label="node-density=enabled"
     label_node_with_label $label

workloads/kube-burner/workloads/node-pod-density/node-pod-density.yml

@@ -81,5 +81,5 @@ jobs:
       - objectTemplate: pod.yml
         replicas: 1
         inputVars:
-          containerImage: gcr.io/google_containers/pause:3.1
+          containerImage: ${NODE_POD_DENSITY_IMAGE}
           nodeSelector: "${POD_NODE_SELECTOR}"

workloads/kube-burner/workloads/node-pod-density/pod.yml

@@ -6,6 +6,10 @@ metadata:
     name: {{.JobName}}
 spec:
   nodeSelector: {{.nodeSelector}}
+  tolerations:
+  - key: os
+    value: Windows
+    Effect: NoSchedule
   containers:
   - name: node-density
     image: {{.containerImage}}