[sn-platform(-slim)] Support readOnlyRootFilesystem (#1136)

* Expose variable KUBECTL_BIN and OUTPUT * Define KUBECTL_BIN and OUTPUT to writable path * Toolset support readOnlyRootFilesystem * Detector support readOnlyRootFilesystem * Fix zk cluster indent * Add rootless example
streamnative · Dec 21, 2023 · 6cd50a0 · 6cd50a0
1 parent 80560ea
commit 6cd50a0
Show file tree

Hide file tree

Showing 34 changed files with 284 additions and 79 deletions.
diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/clean_tls.sh
@@ -81,22 +81,22 @@ done
 
 function delete_ca() {
     local tls_ca_secret="${release}-ca-tls"
-    /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace}
+    ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace}
 }
 
 function delete_server_cert() {
     local component=$1
     local server_cert_secret="${release}-tls-${component}"
 
-    /pulsar/kubectl delete secret ${server_cert_secret} \
+    ${KUBECTL_BIN} delete secret ${server_cert_secret} \
         -n ${namespace}
 }
 
 function delete_client_cert() {
     local component=$1
     local client_cert_secret="${release}-tls-${component}"
 
-    /pulsar/kubectl delete secret ${client_cert_secret} \
+    ${KUBECTL_BIN} delete secret ${client_cert_secret} \
         -n ${namespace}
 }
 

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/cleanup_helm_release.sh
@@ -73,15 +73,15 @@ release=${release:-pulsar-dev}
 
 function delete_namespace() {
     if [[ "${delete_namespace}" == "true" ]]; then
-        /pulsar/kubectl delete namespace ${namespace}
+        ${KUBECTL_BIN} delete namespace ${namespace}
     fi
 }
 
 # delete the cc admin secrets
-/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret
+${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret
 
 # delete tokens
-/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace}
+${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace}
 
 # delete namespace
 delete_namespace
diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform-slim/conf/toolset/pulsar/common_auth.sh
@@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then
     exit 1
 fi
 
-OUTPUT=${CHART_HOME}/output
 OUTPUT_BIN=${OUTPUT}/bin
 PULSARCTL_VERSION=v2.10.2.2
 PULSARCTL_BIN=/pulsar/bin/pulsarctl

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh b/charts/sn-platform-slim/conf/toolset/pulsar/decommission_bookies.sh
@@ -89,8 +89,8 @@ autorecovery_pod=${autorecovery_pod:-autorecovery}
 for ((i=replicas; i>=1; i--))
 do
     j=$((i-1))
-    echo /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset}
-    /pulsar/kubectl -n ${namespace} scale --replicas=${j} sts/${statefulset}
-    echo /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181
-    /pulsar/kubectl -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181
+    echo ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset}
+    ${KUBECTL_BIN} -n ${namespace} scale --replicas=${j} sts/${statefulset}
+    echo ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181
+    ${KUBECTL_BIN} -n ${autorecovery_namespace} exec -it ${autorecovery_pod} -- bin/bookkeeper shell decommissionbookie -bookieid ${statefulset}-${j}.${statefulset}.${namespace}.svc.cluster.local:3181
 done
diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token.sh
@@ -96,11 +96,11 @@ function pulsar::jwt::generate_symmetric_token() {
     trap "test -f $tmpfile && rm $tmpfile" RETURN
     tokentmpfile=$(mktemp)
     trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN
-    /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile}
+    ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['SECRETKEY']}" | base64 --decode > ${tmpfile}
     ${PULSARCTL_BIN} token create -a HS256 --secret-key-file ${tmpfile} --subject ${role} 2&> ${tokentmpfile}
     newtokentmpfile=$(mktemp)
     tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile}
-    /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric"
+    ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=symmetric"
 }
 
 function pulsar::jwt::generate_asymmetric_token() {
@@ -111,11 +111,11 @@ function pulsar::jwt::generate_asymmetric_token() {
     trap "test -f $privatekeytmpfile && rm $privatekeytmpfile" RETURN
     tokentmpfile=$(mktemp)
     trap "test -f $tokentmpfile && rm $tokentmpfile" RETURN
-    /pulsar/kubectl get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile}
+    ${KUBECTL_BIN} get -n ${namespace} secrets ${secret_name} -o jsonpath="{.data['PRIVATEKEY']}" | base64 --decode > ${privatekeytmpfile}
     ${PULSARCTL_BIN} token create -a RS256 --private-key-file ${privatekeytmpfile} --subject ${role} 2&> ${tokentmpfile}
     newtokentmpfile=$(mktemp)
     tr -d '\n' < ${tokentmpfile} > ${newtokentmpfile}
-    /pulsar/kubectl create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric"
+    ${KUBECTL_BIN} create secret generic ${token_name} -n ${namespace} --from-file="TOKEN=${newtokentmpfile}" --from-literal="TYPE=asymmetric"
 }
 
 if [[ "${symmetric}" == "true" ]]; then

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh b/charts/sn-platform-slim/conf/toolset/pulsar/generate_token_secret_key.sh
@@ -18,8 +18,7 @@
 # under the License.
 #
 
-set -e
-
+set -x;
 CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
 cd ${CHART_HOME}
 
@@ -82,9 +81,9 @@ function pulsar::jwt::generate_symmetric_key() {
     tmpfile=$(mktemp)
     trap "test -f $tmpfile && rm $tmpfile" RETURN
     ${PULSARCTL_BIN} token create-secret-key --output-file ${tmpfile}
-    mv $tmpfile SECRETKEY
-    /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=SECRETKEY
-    rm SECRETKEY
+    mv $tmpfile ${OUTPUT}/SECRETKEY
+    ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=${OUTPUT}/SECRETKEY
+    rm ${OUTPUT}/SECRETKEY
 }
 
 function pulsar::jwt::generate_asymmetric_key() {
@@ -95,11 +94,11 @@ function pulsar::jwt::generate_asymmetric_key() {
     publickeytmpfile=$(mktemp)
     trap "test -f $publickeytmpfile && rm $publickeytmpfile" RETURN
     ${PULSARCTL_BIN} token create-key-pair -a RS256 --output-private-key ${privatekeytmpfile} --output-public-key ${publickeytmpfile}
-    mv $privatekeytmpfile PRIVATEKEY
-    mv $publickeytmpfile PUBLICKEY
-    /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} --from-file=PRIVATEKEY --from-file=PUBLICKEY
-    rm PRIVATEKEY
-    rm PUBLICKEY
+    mv $privatekeytmpfile $OUTPUT/PRIVATEKEY
+    mv $publickeytmpfile $OUTPUT/PUBLICKEY
+    ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} --from-file=$OUTPUT/PRIVATEKEY --from-file=$OUTPUT/PUBLICKEY
+    rm $OUTPUT/PRIVATEKEY
+    rm $OUTPUT/PUBLICKEY
 }
 
 if [[ "${symmetric}" == "true" ]]; then

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh b/charts/sn-platform-slim/conf/toolset/pulsar/get_token.sh
@@ -84,8 +84,8 @@ release=${release:-pulsar-dev}
 function pulsar::jwt::get_token() {
     local token_name="${release}-token-${role}"
 
-    local token=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode)
-    local token_type=$(/pulsar/kubectl get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode)
+    local token=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TOKEN']}" | base64 --decode)
+    local token_type=$(${KUBECTL_BIN} get -n ${namespace} secrets ${token_name} -o jsonpath="{.data['TYPE']}" | base64 --decode)
 
     echo "token type: ${token_type}"
     echo "-------------------------"

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh b/charts/sn-platform-slim/conf/toolset/pulsar/gke_bootstrap_script.sh
@@ -58,7 +58,7 @@ function bootstrap(){
 
   echo "Wait for metrics API service"
   # Helm 2.15 and 3.0 bug https://github.com/helm/helm/issues/6361#issuecomment-550503455
-  /pulsar/kubectl --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io
+  ${KUBECTL_BIN} --namespace=kube-system wait --for=condition=Available --timeout=5m apiservices/v1beta1.metrics.k8s.io
 
   helm repo update
 }

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh b/charts/sn-platform-slim/conf/toolset/pulsar/prepare_helm_release.sh
@@ -18,6 +18,7 @@
 # under the License.
 #
 
+set -x;
 CHART_HOME=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/../.. && pwd)
 cd ${CHART_HOME}
 
@@ -94,15 +95,15 @@ pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-ma
 
 function generate_gcs_offloader_service_account_keyfile() {
     local secret_name="${release}-gcs-offloader-service-account"
-    /pulsar/kubectl create secret generic ${secret_name} -n ${namespace} \
+    ${KUBECTL_BIN} create secret generic ${secret_name} -n ${namespace} \
         --from-file="gcs.json=${gcs_offloader_service_account_keyfile}"
 }
 
 pulsar_superusers=${pulsar_superusers:-"proxy-admin,broker-admin,admin,pulsar-manager-admin"}
 
 function do_create_namespace() {
     if [[ "${create_namespace}" == "true" ]]; then
-        /pulsar/kubectl create namespace ${namespace}
+        ${KUBECTL_BIN} create namespace ${namespace}
     fi
 }
 

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh b/charts/sn-platform-slim/conf/toolset/pulsar/setup-clouddns-resolver-service-account.sh
@@ -46,7 +46,7 @@ gcloud iam service-accounts keys create ${RESOLVER_NAME}-key.json \
    --iam-account ${RESOLVER_NAME}@$PROJECT_ID.iam.gserviceaccount.com
 
 echo "Save the service account key as a kubernete secret '${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct' in namespace '${NAMESPACE}'."
-/pulsar/kubectl create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \
+${KUBECTL_BIN} create secret generic ${HELM_RELEASE}-${RESOLVER_NAME}-svc-acct \
    --from-file=${RESOLVER_NAME}-key.json -n ${NAMESPACE}
 
 echo "Remove the generated key."

diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload-lets-encrypt-ca.sh
@@ -29,5 +29,5 @@ PEM="${CA_NAME}.pem"
 
 NAMESPACE=$1
 
-/pulsar/kubectl create secret generic ${CA_NAME}  \
+${KUBECTL_BIN} create secret generic ${CA_NAME}  \
    --from-file=${PEM} -n ${NAMESPACE}
diff --git a/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh b/charts/sn-platform-slim/conf/toolset/pulsar/upload_tls.sh
@@ -91,7 +91,7 @@ ca_cert_file=${tlsdir}/certs/ca.cert.pem
 
 function upload_ca() {
     local tls_ca_secret="${release}-ca-tls"
-    /pulsar/kubectl create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}"
+    ${KUBECTL_BIN} create secret generic ${tls_ca_secret} -n ${namespace} --from-file="ca.crt=${ca_cert_file}"
 }
 
 function upload_server_cert() {
@@ -100,7 +100,7 @@ function upload_server_cert() {
     local tls_cert_file="${tlsdir}/servers/${component}/${component}.cert.pem"
     local tls_key_file="${tlsdir}/servers/${component}/${component}.key-pk8.pem"
 
-    /pulsar/kubectl create secret generic ${server_cert_secret} \
+    ${KUBECTL_BIN} create secret generic ${server_cert_secret} \
         -n ${namespace} \
         --from-file="tls.crt=${tls_cert_file}" \
         --from-file="tls.key=${tls_key_file}" \
@@ -113,7 +113,7 @@ function upload_client_cert() {
     local tls_cert_file="${tlsdir}/clients/${component}/${component}.cert.pem"
     local tls_key_file="${tlsdir}/clients/${component}/${component}.key-pk8.pem"
 
-    /pulsar/kubectl create secret generic ${client_cert_secret} \
+    ${KUBECTL_BIN} create secret generic ${client_cert_secret} \
         -n ${namespace} \
         --from-file="tls.crt=${tls_cert_file}" \
         --from-file="tls.key=${tls_key_file}" \

diff --git a/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml b/charts/sn-platform-slim/templates/detector/pulsar-detector-deployment.yaml
@@ -85,6 +85,9 @@ spec:
         {{- if .Values.pulsar_detector.resources }}
         resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }}
         {{- end }}
+        volumeMounts:
+        - name: tmp
+          mountPath: /pulsar/logs
       # This init container will wait for at least one broker to be ready before
       # deploying the pulsar-detector
       - name: wait-broker-ready
@@ -103,6 +106,9 @@ spec:
         {{- if .Values.pulsar_detector.resources }}
         resources: {{- toYaml .Values.pulsar_detector.resources | nindent 10 }}
         {{- end }}
+        volumeMounts:
+        - name: tmp
+          mountPath: /pulsar/logs
       {{- end }}
       containers:
       - name: "{{ template "pulsar.fullname" . }}-{{ .Values.pulsar_detector.component }}"
@@ -145,8 +151,10 @@ spec:
         volumeMounts: 
         {{- toYaml .Values.pulsar_detector.extraVolumeMounts | nindent 10 }}
         {{- end }}
-      {{- if .Values.pulsar_detector.extraVolumes }}
       volumes:
+      - name: tmp
+        emptyDir: {}
+      {{- if .Values.pulsar_detector.extraVolumes }}
       {{- toYaml .Values.pulsar_detector.extraVolumes | nindent 8 }}
       {{- end }}
 {{- end }}
diff --git a/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml b/charts/sn-platform-slim/templates/toolset/jwt-secret-init-job.yaml
@@ -79,13 +79,14 @@ spec:
         args:
           - |
             set -ex;
-            cp /tmp/binaries/kubectl /pulsar/kubectl;
-            chmod +x /pulsar/kubectl;
             mkdir -p scripts/pulsar;
             cp scripts/jwt-secret-config/* scripts/pulsar;
             chmod +x scripts/pulsar/*;
             usingSecretKey={{ .Values.auth.authentication.jwt.usingSecretKey }};
             ls -lh scripts/pulsar/;
+            export KUBECTL_BIN=/tmp/binaries/kubectl;
+            export OUTPUT=scripts/pulsar/output;
+            mkdir ${OUTPUT};
             if [ "${usingSecretKey}" = "true" ]; then
               ./scripts/pulsar/prepare_helm_release.sh -n {{ template "pulsar.namespace" . }} -k {{ .Release.Name }} --symmetric;
             else 

diff --git a/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml b/charts/sn-platform-slim/templates/toolset/toolset-statefulset.yaml
@@ -72,8 +72,27 @@ spec:
 {{ toYaml .Values.toolset.tolerations | indent 8 }}
     {{- end }}
       terminationGracePeriodSeconds: {{ .Values.toolset.gracePeriod }}
-    {{- if .Values.toolset.installBusybox }}
       initContainers:
+      {{- if .Values.toolset.readOnlyRootFilesystem }}
+      - name: "init-copy-config"
+        image: "{{ .Values.images.toolset.repository }}:{{ .Values.images.toolset.tag }}"
+        imagePullPolicy: {{ .Values.images.toolset.pullPolicy }}
+        command:
+        - sh
+        - -c
+        - |
+          set -ex;
+          cp -r /pulsar/conf/* /conf_tmp/;
+          echo OK > /conf_tmp/status;
+        {{- if .Values.toolset.resources }}
+        resources:
+{{ toYaml .Values.toolset.resources | indent 10 }}
+      {{- end }}
+        volumeMounts:
+         - name: tmp
+           mountPath: /conf_tmp/
+      {{- end }}
+      {{- if .Values.toolset.installBusybox }}
       - name: busybox
         image: "{{ .Values.images.toolset.busybox.repository }}:{{ .Values.images.toolset.busybox.tag }}"
         imagePullPolicy: {{ .Values.images.toolset.busybox.pullPolicy }}
@@ -98,7 +117,7 @@ spec:
         volumeMounts:
         - name: binaries
           mountPath: /home/tmp/binaries
-    {{- end }}
+      {{- end }}
       containers:
       - name: "pulsar"
         {{- include "pulsar.toolset.image" . | nindent 8 }}
@@ -122,6 +141,12 @@ spec:
 {{ toYaml . | indent 8 }}
 {{- end }}
         volumeMounts:
+        {{- if .Values.toolset.readOnlyRootFilesystem }}
+        - name: tmp
+          mountPath: /pulsar/conf
+        - name: tmp
+          mountPath: /pulsar/logs
+        {{- end }}
         {{- if .Values.toolset.installBusybox }}
         - name: binaries
           mountPath: /bin/busybox
@@ -138,6 +163,10 @@ spec:
 {{ toYaml . | indent 8 }}
 {{- end }}
       volumes:
+      {{- if .Values.toolset.readOnlyRootFilesystem }}
+      - name: tmp
+        emptyDir: {}
+      {{- end }}
       {{- if .Values.toolset.installBusybox }}
       - name: binaries
         emptyDir: {}

diff --git a/charts/sn-platform-slim/values.yaml b/charts/sn-platform-slim/values.yaml
@@ -1550,6 +1550,7 @@ toolset:
   component: toolset
   useProxy: false
   installBusybox: true
+  readOnlyRootFilesystem: false
   replicaCount: 1
   # nodeSelector:
     # cloud.google.com/gke-nodepool: default-pool

diff --git a/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh b/charts/sn-platform/conf/toolset/pulsar/clean_tls.sh
@@ -81,22 +81,22 @@ done
 
 function delete_ca() {
     local tls_ca_secret="${release}-ca-tls"
-    /pulsar/kubectl delete secret ${tls_ca_secret} -n ${namespace}
+    ${KUBECTL_BIN} delete secret ${tls_ca_secret} -n ${namespace}
 }
 
 function delete_server_cert() {
     local component=$1
     local server_cert_secret="${release}-tls-${component}"
 
-    /pulsar/kubectl delete secret ${server_cert_secret} \
+    ${KUBECTL_BIN} delete secret ${server_cert_secret} \
         -n ${namespace}
 }
 
 function delete_client_cert() {
     local component=$1
     local client_cert_secret="${release}-tls-${component}"
 
-    /pulsar/kubectl delete secret ${client_cert_secret} \
+    ${KUBECTL_BIN} delete secret ${client_cert_secret} \
         -n ${namespace}
 }
 

diff --git a/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh b/charts/sn-platform/conf/toolset/pulsar/cleanup_helm_release.sh
@@ -73,15 +73,15 @@ release=${release:-pulsar-dev}
 
 function delete_namespace() {
     if [[ "${delete_namespace}" == "true" ]]; then
-        /pulsar/kubectl delete namespace ${namespace}
+        ${KUBECTL_BIN} delete namespace ${namespace}
     fi
 }
 
 # delete the cc admin secrets
-/pulsar/kubectl delete -n ${namespace} secret ${release}-admin-secret
+${KUBECTL_BIN} delete -n ${namespace} secret ${release}-admin-secret
 
 # delete tokens
-/pulsar/kubectl get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs /pulsar/kubectl delete secrets -n ${namespace}
+${KUBECTL_BIN} get secrets -n ${namespace} | grep ${release}-token- | awk '{print $1}' | xargs ${KUBECTL_BIN} delete secrets -n ${namespace}
 
 # delete namespace
 delete_namespace
diff --git a/charts/sn-platform/conf/toolset/pulsar/common_auth.sh b/charts/sn-platform/conf/toolset/pulsar/common_auth.sh
@@ -23,7 +23,6 @@ if [ -z "$CHART_HOME" ]; then
     exit 1
 fi
 
-OUTPUT=${CHART_HOME}/output
 OUTPUT_BIN=${OUTPUT}/bin
 PULSARCTL_VERSION=v2.10.2.2
 PULSARCTL_BIN=/pulsar/bin/pulsarctl