Disable validation-webhook daemonset for managed clusters

When running workloads on managed clusters with cluster-admin user, we are seeing an error like "admission webhook
"regular-user-validation.managed.openshift.io" denied the request" when trying add a label to a
node directly by using oc label node. The recommended way to add labels
to nodes on managed ROSA clusters is by editing the machinepool.
However, the Default machinepool cannot be edited to add labels and we
see an error like "Labels cannot be updated on the Default machine pool".
The only way to add a label is by disabling the
validation-webhook daemonset and thereby admission control in the
openshift-validation-webhook project that only exists on managed services
clusters. We disable the daemonset by adding a fake nodeSelector before
labeling the nodes and remove the nodeSelector after unlabeling the nodes.
Adding a nodeSelector on top of the existing nodeAffinity means that both
the conditions needs to be met for a pod to be scheduled. Also by adding
the nodeSelector, the spec is not overwritten during reconcillation whereas
changes to nodeAffinity are being overwritten.

This change4 is important for managed clusters as we don't always have access to kubeconfig (when
running in prow for example).

Signed-off-by: Sai Sindhur Malleni <[email protected]>

workloads/kube-burner-ocp-wrapper/run.sh

@@ -14,11 +14,32 @@ EXTRA_FLAGS=${EXTRA_FLAGS:-}
 UUID=$(uuidgen)
 KUBE_DIR=${KUBE_DIR:-/tmp}
 
+check_managed_cluster() {
+   status=$(oc get infrastructure/cluster -o=jsonpath='{.status.platformStatus.*.resourceTags[0]}')
+   if [[ $status =~ managed ]]; then
+      echo "Detected a Managed Cluster"
+      managed=true
+   fi
+}
+
+remove_managed_webhook_validation() {
+   echo "Disabling validation-webhook for Managed cluster"
+   oc patch -n openshift-validation-webhook daemonset validation-webhook -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
+
+}
+
+add_managed_webhook_validation() {
+    echo "Enabling validation-webhook for Managed cluster"
+    oc patch -n openshift-validation-webhook daemonset validation-webhook --type json -p '[{ "op": "remove", "path": "/spec/template/spec/nodeSelector" }]'
+}
+
 download_binary(){
   KUBE_BURNER_URL=https://github.com/cloud-bulldozer/kube-burner/releases/download/v${KUBE_BURNER_VERSION}/kube-burner-${KUBE_BURNER_VERSION}-Linux-x86_64.tar.gz
   curl -sS -L ${KUBE_BURNER_URL} | tar -xzC ${KUBE_DIR}/ kube-burner
 }
 
+check_managed_cluster
+
 hypershift(){
   echo "HyperShift detected"
   # Get hosted cluster ID and name
@@ -96,5 +117,13 @@ EOF
 )
 curl -k -sS -X POST -H "Content-type: application/json" ${ES_SERVER}/ripsaw-kube-burner/_doc -d "${METADATA}" -o /dev/null
 
+if [[ $managed == true ]]; then
+   remove_managed_webhook_validation
+fi
+
 echo $cmd
-exec $cmd
+$cmd
+
+if [[ $managed == true ]]; then
+   add_managed_webhook_validation
+fi

workloads/kube-burner/common.sh

@@ -94,6 +94,25 @@ run_workload() {
   gen_metadata ${WORKLOAD} ${start_date} $(date +%s%3N)
 }
 
+check_managed_cluster() {
+   status=$(oc get infrastructure/cluster -o=jsonpath='{.status.platformStatus.*.resourceTags[0]}')
+   if [[ $status =~ managed ]]; then
+      log "Detected a Managed Cluster"
+      managed=true
+   fi
+}
+
+remove_managed_webhook_validation() {
+   log "Disabling validation-webhook for Managed cluster"
+   oc patch -n openshift-validation-webhook daemonset validation-webhook -p '{"spec": {"template": {"spec": {"nodeSelector": {"non-existing": "true"}}}}}'
+
+}
+
+add_managed_webhook_validation() {
+    log "Enabling validation-webhook for Managed cluster"
+    oc patch -n openshift-validation-webhook daemonset validation-webhook --type json -p '[{ "op": "remove", "path": "/spec/template/spec/nodeSelector" }]'
+}
+
 find_running_pods_num() {
   pod_count=0
   # The next statement outputs something similar to:

workloads/kube-burner/run.sh

@@ -5,6 +5,12 @@
 . ../../utils/compare.sh
 
 label=""
+
+check_managed_cluster
+if [[ $managed == true ]]; then
+   remove_managed_webhook_validation
+fi
+
 case ${WORKLOAD} in
   cluster-density)
     WORKLOAD_TEMPLATE=workloads/cluster-density/cluster-density.yml
@@ -163,6 +169,10 @@ else
   run_workload
 fi
 
+if [[ $managed == true ]]; then
+   add_managed_webhook_validation
+fi
+
 if [[ ${CLEANUP_WHEN_FINISH} == "true" ]]; then
   cleanup
   if [[ ${WORKLOAD} == node-density* || ${WORKLOAD} == pod-density-heavy ]]; then