update e2e to import hosted cluster (#17)

Signed-off-by: Zhiwei Yin <[email protected]>

.github/workflows/e2e.yaml

@@ -28,19 +28,48 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GO_VERSION }}
-      - name: Setup kind
-        uses: engineerd/setup-kind@v0.5.0
+      - name: Install kind
+        uses: helm/kind[email protected].0
         with:
           version: v0.22.0
-          skipClusterCreation: ${{ env.USE_EXISTING_CLUSTER }}
-      - name: Set KUBECONFIG
+          install_only: true
+      - name: Test kind works and there is no cluster started
+        run: |
+          [[ $(kind get clusters | wc -l) -eq 0 ]]
+      - name: Set KUBECONFIG dir
         run: |
           mkdir -p /home/runner/.kube
-          kind get kubeconfig > /home/runner/.kube/config
-        if: ${{ env.USE_EXISTING_CLUSTER }}
-      - name: Test E2E
+      - name: Create Hub cluster
+        run: |
+          kind create cluster --name hub
+        env:
+          KUBECONFIG: /home/runner/.kube/hub-kubeconfig      
+      - name: Create Spoke cluster
+        run: |
+          kind create cluster --name spoke
+        env:
+          KUBECONFIG: /home/runner/.kube/spoke-kubeconfig                     
+      - name: Test Hub and Spoke clusters
+        run: |
+          kubectl cluster-info --kubeconfig /home/runner/.kube/hub-kubeconfig 
+          kubectl get pods -A --kubeconfig /home/runner/.kube/hub-kubeconfig 
+          kubectl cluster-info --kubeconfig /home/runner/.kube/spoke-kubeconfig 
+          kubectl get pods -A --kubeconfig /home/runner/.kube/spoke-kubeconfig 
+      - name: Create internal kubeconfigs
+        run: |
+          kind get kubeconfig --internal --name hub > /home/runner/.kube/hub-internal-kubeconfig 
+          kind get kubeconfig --internal --name spoke > /home/runner/.kube/spoke-internal-kubeconfig 
+      - name: Install MCE+Policy
+        run: |
+          make e2e-install
+        env:
+          KUBECONFIG: /home/runner/.kube/hub-kubeconfig 
+
+      - name: Import hosted cluster
         run: |
-          make test-e2e
+          make e2e-import-cluster
         env:
-          KUBECONFIG: /home/runner/.kube/config
-
+          KUBECONFIG: /home/runner/.kube/hub-kubeconfig 
+          MANAGED_CLUSTER_NAME: spoke
+          MANAGED_KUBECONFIG: /home/runner/.kube/spoke-internal-kubeconfig 
+          EXTERNAL_MANAGED_KUBECONFIG: /home/runner/.kube/spoke-internal-kubeconfig 

Makefile

@@ -35,8 +35,11 @@ install-mce: ensure-helm
 install-policy: ensure-helm
 	$(HELM) install policy ./policy
 
-test-e2e: 
-	hack/e2e.sh
+e2e-install:
+	hack/e2e-install.sh 
+
+e2e-import-cluster: 
+	hack/e2e-import-cluster.sh
 
 ensure-helm:
 ifeq "" "$(wildcard $(HELM))"

configuration/klusterletconfig.yaml

@@ -3,4 +3,4 @@ kind: KlusterletConfig
 metadata:
   name: global
 spec:
-  hubKubeAPIServerURL: "https://kind-control-plane:6443"
+  hubKubeAPIServerURL: "hub cluster api server url"

e2e/configuration/cluster.yaml

@@ -0,0 +1,14 @@
+apiVersion: cluster.open-cluster-management.io/v1
+kind: ManagedCluster
+metadata:
+  name: spoke
+  annotations:
+    import.open-cluster-management.io/klusterlet-deploy-mode: Hosted
+    import.open-cluster-management.io/hosting-cluster-name: local-cluster
+    addon.open-cluster-management.io/enable-hosted-mode-addons: "true"
+    open-cluster-management/created-via: other
+  labels:
+    cluster.open-cluster-management.io/clusterset: default
+spec:
+  hubAcceptsClient: true
+  leaseDurationSeconds: 60

e2e/configuration/klusterletconfig.yaml

@@ -0,0 +1,6 @@
+apiVersion: config.open-cluster-management.io/v1alpha1
+kind: KlusterletConfig
+metadata:
+  name: global
+spec:
+  hubKubeAPIServerURL: "https://hub-control-plane:6443"

e2e/configuration/mce-values.yaml

@@ -0,0 +1,31 @@
+
+replicaCount: 1
+availabilityConfig: Basic
+
+images:
+  overrides:
+    backplane_operator: "quay.io/stolostron/backplane-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    registration_operator: "quay.io/stolostron/registration-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    hypershift_addon_operator: "quay.io/stolostron/hypershift-addon-operator:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    managedcluster_import_controller: "quay.io/stolostron/managedcluster-import-controller:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    multicloud_manager: "quay.io/stolostron/multicloud-manager:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    addon_manager: "quay.io/stolostron/addon-manager:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    work: "quay.io/stolostron/work:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    registration: "quay.io/stolostron/registration:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    placement: "quay.io/stolostron/placement:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+    kube_rbac_proxy_mce: "quay.io/stolostron/kube-rbac-proxy-mce:2.7.0-BACKPLANE-2024-09-30-05-39-18"
+
+
+    #images in MCE 2.6.2
+    # backplane_operator: "registry.redhat.io/multicluster-engine/backplane-rhel9-operator@sha256:eb15286f728e32851b426f86b3a1ce5ed186cdb1a67287e98ff5925dc558a2a9"
+    # registration_operator: "registry.redhat.io/multicluster-engine/registration-operator-rhel9@sha256:efe0091dd6d389190c0acf47cb8980ed8ba7d90bdf04b3b2e9dc07926c0d0bca"
+    # hypershift_addon_operator: "registry.redhat.io/multicluster-engine/hypershift-addon-rhel9-operator@sha256:7b375b10a5f6434aad3801e486e51a9404d88899a84593686856f3340a5889fa"
+    # managedcluster_import_controller: "registry.redhat.io/multicluster-engine/managedcluster-import-controller-rhel9@sha256:89dea3fa0cc7cb182d49436bb93214af73bac1383caa59766c97b6fdad2b14b5"
+    # multicloud_manager: "registry.redhat.io/multicluster-engine/multicloud-manager-rhel9@sha256:089188b25407a49ba042bd65a8afc6a6c86d68b8d7ac3da722196f0cd85383ae"
+    # addon_manager: "registry.redhat.io/multicluster-engine/addon-manager-rhel9@sha256:93b21a4356230da6f70edac4d8770b5e0af2f40a37ec454b00b12b5aa76fb4c3"
+    # work: "registry.redhat.io/multicluster-engine/work-rhel9@sha256:bf6c7384283046093605659e866ed6a4e2c7c81690f5180894099d4691b97aa3"
+    # registration: "registry.redhat.io/multicluster-engine/registration-rhel9@sha256:e05fe0fb10bd9abd3f653fc01ea25dab0e78b3dbe932b2da3375585dffc7f4b6"
+    # placement: "registry.redhat.io/multicluster-engine/placement-rhel9@sha256:307b984b4315b9e22c520be893c2522d4bf3090cd90b10ba1aefb740bc7b7cc2"
+    # kube_rbac_proxy_mce: "registry.redhat.io/multicluster-engine/kube-rbac-proxy-mce-rhel9@sha256:b5f6f36487e70b543afacd33255b0c4f504bbd62bd5b7f46648c098db4466393"
+  imageCredentials:
+    dockerConfigJson: ""

e2e/configuration/multiclusterengine.yaml

@@ -0,0 +1,40 @@
+apiVersion: multicluster.openshift.io/v1
+kind: MultiClusterEngine
+metadata:
+  name: multiclusterengine
+spec:
+  availabilityConfig: Basic
+  imagePullSecret: open-cluster-management-image-pull-credentials
+  overrides:
+    components:
+      - enabled: true
+        name: local-cluster
+      - enabled: true
+        name: cluster-manager
+      - enabled: true
+        name: server-foundation
+      - enabled: true
+        name: hypershift-local-hosting
+      - enabled: true
+        name: hypershift
+      - enabled: false
+        name: cluster-lifecycle
+      - enabled: false
+        name: discovery
+      - enabled: false
+        name: console-mce
+      - enabled: false
+        name: hive
+      - enabled: false
+        name: assisted-service
+      - enabled: false
+        name: image-based-install-operator-preview
+      - enabled: false
+        name: cluster-proxy-addon
+      - enabled: false
+        name: managedserviceaccount
+  targetNamespace: multicluster-engine
+  tolerations:
+    - effect: NoSchedule
+      key: node-role.kubernetes.io/infra
+      operator: Exists

e2e/configuration/policy-values.yaml

@@ -0,0 +1,28 @@
+global:
+  imageOverrides:
+    # upstream images
+    governance_policy_propagator: "quay.io/stolostron/governance-policy-propagator:2.12.0-SNAPSHOT-2024-09-30-01-46-06"
+    governance_policy_addon_controller: "quay.io/stolostron/governance-policy-addon-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06" 
+    config_policy_controller: "quay.io/stolostron/config-policy-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06"
+    governance_policy_framework_addon: "quay.io/stolostron/governance-policy-framework-addon:2.12.0-SNAPSHOT-2024-09-30-01-46-06"
+    klusterlet_addon_controller: "quay.io/stolostron/klusterlet-addon-controller:2.12.0-SNAPSHOT-2024-09-30-01-46-06"
+
+    # images in ACM 2.11.2
+    # governance_policy_propagator: "registry.redhat.io/rhacm2/governance-policy-propagator-rhel9@sha256:af848e7e31d8ec9b5ad1896a5d5ccc67f320a7740245c190ba8a76757984e65b"
+    # governance_policy_addon_controller: "registry.redhat.io/rhacm2/acm-governance-policy-addon-controller-rhel9@sha256:fc0708f0a6d5266fb544f41b61d9697d370c8c5e297e4e3f13de8656f9c2b049" 
+    # config_policy_controller: "registry.redhat.io/rhacm2/config-policy-controller-rhel9@sha256:cecf914d7fb7759a4f512c1ec53a077dcb1c7e405c22a5bf6af1bf5878cf3c42"
+    # governance_policy_framework_addon: "registry.redhat.io/rhacm2/acm-governance-policy-framework-addon-rhel9@sha256:a4880f6e82d2b82606203ea855d0418bb29b3d4535f8bc7a9ef4074258c18674"
+    # klusterlet_addon_controller: "registry.redhat.io/rhacm2/klusterlet-addon-controller-rhel9@sha256:478e3e6cda0d74f43b0f05911d023344108a5cd79d57d5cc9f268ad064848a00"
+  namespace: multicluster-engine
+  pullSecret: open-cluster-management-image-pull-credentials
+
+  cma:
+    defaultConfig:
+      name: addon-hosted-config
+      namespace: multicluster-engine
+grc:
+  hubconfig:
+    replicaCount: 1
+cluster-lifecycle:
+  hubconfig:
+    replicaCount: 1

e2e/mce-chart/templates/multiclusterengine.yaml

@@ -3,7 +3,7 @@ kind: MultiClusterEngine
 metadata:
   name: multiclusterengine
 spec:
-  availabilityConfig: High
+  availabilityConfig: {{ .Values.availabilityConfig }}
   imagePullSecret: open-cluster-management-image-pull-credentials
   overrides:
     components:

e2e/mce-chart/values.yaml

@@ -1,5 +1,6 @@
 
-replicaCount: 1
+replicaCount: 2
+availabilityConfig: High
 images:
 
   overrides:

hack/e2e-import-cluster.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# env
+# export KUBECONFIG=kubeconfig
+# export MANAGED_KUBECONFIG=managed-kubeconfig
+# export EXTERNAL_MANAGED_KUBECONFIG=external-managed-kubeconfig
+# export MANAGED_CLUSTER_NAME=spoke
+
+KLUSTERLET_NS=klusterlet-$MANAGED_CLUSTER_NAME
+
+function waitForReady() {
+    FOUND=1
+    SECOND=0
+    local cmd="$1"
+    local rst="$2"
+
+    echo "check if \"$cmd\" ==  $rst"
+
+    while [ ${FOUND} -eq 1 ]; do
+        if [ $SECOND -gt 240 ]; then
+            echo "Timeout waiting for the result of cmd. "
+
+            kubectl get pods -A
+            kubectl get mcl
+            kubectl get mce multiclusterengine -o yaml
+
+            exit 1
+        fi
+
+        result=$(bash -c "$cmd" 2>/dev/null || true) 
+        if [ "$rst" -eq "$result" ]; then
+            echo "pass "
+            break
+        fi
+
+        echo "expected $rst, but got $result, re-try after 5 seconds..."
+        sleep 5
+        (( SECOND = SECOND + 5 ))
+    done
+}
+
+
+echo ""
+echo "###### Create managed cluster ns ###### "
+result=$(kubectl get ns | grep -c $MANAGED_CLUSTER_NAME  2>/dev/null || true)
+if [ $result -eq 0 ] ; then 
+kubectl create ns $MANAGED_CLUSTER_NAME; 
+fi
+
+echo ""
+echo "###### Create auto-import secret"
+result=$(kubectl get secret -n $MANAGED_CLUSTER_NAME | grep -c auto-import-secret  2>/dev/null || true)
+if [ $result -eq 0 ]; then 
+  kubectl create secret generic auto-import-secret --from-file=kubeconfig=$MANAGED_KUBECONFIG -n $MANAGED_CLUSTER_NAME
+fi
+
+echo ""
+echo "###### Create managedCluster ######"
+cat << EOF | kubectl apply -f -
+apiVersion: cluster.open-cluster-management.io/v1
+kind: ManagedCluster
+metadata:
+  name: $MANAGED_CLUSTER_NAME
+  annotations:
+    import.open-cluster-management.io/klusterlet-deploy-mode: Hosted
+    import.open-cluster-management.io/hosting-cluster-name: local-cluster
+    addon.open-cluster-management.io/enable-hosted-mode-addons: "true"
+    open-cluster-management/created-via: other
+  labels:
+    cluster.open-cluster-management.io/clusterset: default
+spec:
+  hubAcceptsClient: true
+  leaseDurationSeconds: 60
+EOF
+
+echo ""
+echo "###### Create external-managed-kubeconfig secret ######"
+waitForReady "kubectl get ns | grep -c \"klusterlet-$MANAGED_CLUSTER_NAME\"" 1
+result=$(kubectl get secret -n $KLUSTERLET_NS | grep -c external-managed-kubeconfig  2>/dev/null || true)
+if [ $result -eq 0 ]; then 
+  kubectl create secret generic external-managed-kubeconfig --from-file=kubeconfig=$EXTERNAL_MANAGED_KUBECONFIG -n $KLUSTERLET_NS
+fi
+
+echo ""
+echo "###### Wait for $MANAGED_CLUSTER_NAME is ready ######"
+waitForReady "kubectl get mcl $MANAGED_CLUSTER_NAME | grep -c \"True\"" 1
+
+echo ""
+echo "###### Wait unitl 3 addons in $MANAGED_CLUSTER_NAME is Available ######"
+waitForReady "kubectl get mca -n $MANAGED_CLUSTER_NAME | grep -c \"True\"" 3
+
+echo ""
+echo "!!!!!!!!!! hosted cluster $MANAGED_CLUSTER_NAME is imported succussfully !!!!!!!!!!!!"
+echo ""

hack/e2e.sh → hack/e2e-install.sh

@@ -4,6 +4,10 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# env
+export HELM=_output/bin/helm
+# export KUBECONFIG=kubeconfig
+
 function waitForReady() {
     FOUND=1
     SECOND=0
@@ -37,8 +41,10 @@ function waitForReady() {
 
 
 echo ""
-echo "#### Install MCE ####"
-make install-mce
+echo "#### Install MCE on Hub cluster ####"
+make ensure-helm
+$HELM install mce ./e2e/mce-chart -f ./e2e/configuration/mce-values.yaml
+
 
 echo ""
 echo "###### Wait until MCE pod is running ######"
@@ -57,12 +63,11 @@ waitForReady "kubectl get crds | grep -c \"klusterletconfigs\"" 1
 
 echo ""
 echo "###### Create global klusterletconfig ######"
-# set hub api server to ./configuration/klusterletconfig.yaml
-kubectl apply -f ./configuration/klusterletconfig.yaml
+kubectl apply -f ./e2e/configuration/klusterletconfig.yaml
 
 echo ""
 echo "###### Wait unitl local-cluster is created ######"
-waitForReady "kubectl get mcl  | grep -c \"local-cluster\"" 1
+waitForReady "kubectl get mcl local-cluster | grep -c \"True\"" 1
 
 
 echo ""
@@ -73,10 +78,16 @@ echo ""
 echo "###### create addonhostedconfig ######"
 kubectl apply -f ./configuration/addonhostedconfig.yaml
 
+echo ""
+echo "###### patch clustermanagementaddon work-manager ######"
+#kubectl patch clustermanagementaddon work-manager --type merge -p '{"spec":{"supportedConfigs":[{"defaultConfig":{"name":"addon-hosted-config","namespace":"multicluster-engine"},"group":"addon.open-cluster-management.io","resource":"addondeploymentconfigs"}]}}'
+kubectl apply -f ./configuration/workmanagercma.yaml
 
 echo ""
 echo "#### Install Policy addons #####"
-make install-policy
+make ensure-helm
+$HELM install policy ./policy -f ./e2e/configuration/policy-values.yaml
+
 
 echo ""
 echo "###### Enable policy addons for local-cluster ######"

kubeconfig-kind-spoke-insternal

@@ -0,0 +1,20 @@
+apiVersion: v1
+clusters:
+- cluster:
+    certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURCVENDQWUyZ0F3SUJBZ0lJV2Niak0yQ3RZTWd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFd01UTXhOakEwTkRkYUZ3MHpOREV3TVRFeE5qQTVORGRhTUJVeApFekFSQmdOVkJBTVRDbXQxWW1WeWJtVjBaWE13Z2dFaU1BMEdDU3FHU0liM0RRRUJBUVVBQTRJQkR3QXdnZ0VLCkFvSUJBUUM1b1ltcjhpeTZ0dDd4TW8zRm5sTzcvL2JQOFlncXU5T3MwRWxabGMwYUZ0YXlpeUVNYTNtQ0dxeDUKWXdRWktaQXF3WkRRUCtCNk8wY2RVZGhOQmtjeEx0V0pBdzdzNEtsWGw4OVdzbVJiall3RE0vWCtPN1NyRWE1SApLMXIwY3pHcVhHTThWMTlIcVZhMVU2NlhYMFpaak1MaXpUWDM0Skpnb0llMlFWOHl4ZEgxd2NyMXRHT2JkRUhzCi9FdVlucllSZ2lJb2xYQ2hKQUxSVldNM2dyQnZ6SFduY3FGUHYwOE9hSmhpM0pCcWc4ZXFYbUdxU2Q2NFc1SHEKQklnb0Joa09UeklPbUs2NUFTQjIyUzlwdjVIREgrRTBrNDVUZXRFb0czWjZEWHVwR3FlUkJ6aVptWkVmNUw1VQp6WEZJMzR3OGtiQUtLV1N2ajNKMTJtbm4rU1N4QWdNQkFBR2pXVEJYTUE0R0ExVWREd0VCL3dRRUF3SUNwREFQCkJnTlZIUk1CQWY4RUJUQURBUUgvTUIwR0ExVWREZ1FXQkJURkI3MFQ0enFoSkNiSGhja2lwWUhrSEVmcVJEQVYKQmdOVkhSRUVEakFNZ2dwcmRXSmxjbTVsZEdWek1BMEdDU3FHU0liM0RRRUJDd1VBQTRJQkFRQzRxRkkwNUduMApkZjBPd0o0YXNOcFZsN2d1Y0c4TmcyVjZQN3JSSkt6VFQ4bnpNb1ljbCt0amo4dldHcy84QUhrVUpxWDlUYk53ClZrNmF4YnVzSmdock03MkcyeExsMHo0amRNSXhuT2RqQ1lnajhNdmcycDRyYU1uUDVLWFVpWmRPQUdMeGJ4aC8KMjliemJOR0loQTFlMEpGWE9KV0tia25qcjFubGRyZ2U5T280cXVwWlJ5K3hiR0ZzNkhuMjlGQXNmWHI1K1dJWgpyOVlyVjNhWDh1RTdweThFbDRTU240ZnhINWZMVkgzVVdOdHF3OE54aHh2WXh2cjhvLzlTQnd2Zys3OFVhY1NxClFkWEtIc0l4NVNpVStjT1hMWkp6MFVPQlk2NFNZOXg0bXpZa3V5RTQ4aUFFZFhpTkdQUlFid2tNbkpXRDc4NFMKTGlLSzhBdFR2blpjCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    server: https://spoke-control-plane:6443
+  name: kind-spoke
+contexts:
+- context:
+    cluster: kind-spoke
+    user: kind-spoke
+  name: kind-spoke
+current-context: kind-spoke
+kind: Config
+preferences: {}
+users:
+- name: kind-spoke
+  user:
+    client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURLVENDQWhHZ0F3SUJBZ0lJTmRSRDk2algzaGd3RFFZSktvWklodmNOQVFFTEJRQXdGVEVUTUJFR0ExVUUKQXhNS2EzVmlaWEp1WlhSbGN6QWVGdzB5TkRFd01UTXhOakEwTkRkYUZ3MHlOVEV3TVRNeE5qQTVORGxhTUR3eApIekFkQmdOVkJBb1RGbXQxWW1WaFpHMDZZMngxYzNSbGNpMWhaRzFwYm5NeEdUQVhCZ05WQkFNVEVHdDFZbVZ5CmJtVjBaWE10WVdSdGFXNHdnZ0VpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElCRHdBd2dnRUtBb0lCQVFDeWpWaTYKS0p6MzJwMXhick5ZWEJoUGpxblQvZlExa0I2ZWszTkxuSE5qdHVWZFJlRXZGeEVaTGVad2RoKzJQUWpCZzY2TApjK3Nvenc3RW9OMEwxWHl6Q3lmT0phMHFYeENvbmdyTjNMUk94S3RNamJTeUppUnl2RXNqZW9yU0VYUjgrclh0CktKQ0s3L2JzZ2JNTVJON1JPZVBYbzJGTk5VVkZxdmZYWWZHUTd6RXZtemJoN1FSQ2dPM2E2bGZhUzJXWXErZCsKRXczbjg1aHZQVnVjeStJZVRpWTRDZDMzY1BnOGU5cWsvZEpWNldoNXg3bWZNT1NKS1Evbk9VdHoraWFpWWllTAo4c0pTaGJGcnZ3a2RBUlZuRlRXSHF0TmFMNmdNdzdUaGJuSVl6WWQvTVFPOUowK1dQaUxwT1pKeXlWeTM1VEh6CmFTMVo4eG5rM1ROUU1lUlBBZ01CQUFHalZqQlVNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUsKQmdnckJnRUZCUWNEQWpBTUJnTlZIUk1CQWY4RUFqQUFNQjhHQTFVZEl3UVlNQmFBRk1VSHZSUGpPcUVrSnNlRgp5U0tsZ2VRY1IrcEVNQTBHQ1NxR1NJYjNEUUVCQ3dVQUE0SUJBUUFQUnBtUjlnbWE1STc2cWtIMFllcXl2cmVsCjJ5MjVxcGQyK1ZxSXhhVDliQ3M1eG14Rk9TaHAzVXZUbXFjUzlkZzd1TnNDUERUakZVY2ZjUEd5bCszeGt6ZFkKV24xSmxhZWJOUlFUcjdUZjVyR0dnUk9UVHlYRTBVUmt1QzVMN2thdHZQamNhK0dtSHV3NEp5elVRV2g1VkdVYQpRdXZURHVHeUF3aCt5MGlsRUY1V1pId3lsRFhudVB1Ylo5ZXIzR21ranNzY05TV2lGYXhDMkJZTERHejRYclIzCmVkR3B2ZkJZajllWGhCbjN5OGxZeDdKeGZEb3h1Nm5ZMnJlZjVybG1xck94WmJqd0hrbHpBQmJTNnNaR3ViVnIKY0h5WFg3WDh3ay9rWDZrZzhnR3ZsZy9rWURsQ1pGcEk2ZUZYR29VOXRGbFdxeGY5VnhEL3F1aDVxTDFHCi0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K
+    client-key-data: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBc28xWXVpaWM5OXFkY1c2eldGd1lUNDZwMC8zME5aQWVucE56UzV4elk3YmxYVVhoCkx4Y1JHUzNtY0hZZnRqMEl3WU91aTNQcktNOE94S0RkQzlWOHN3c256aVd0S2w4UXFKNEt6ZHkwVHNTclRJMjAKc2lZa2NyeExJM3FLMGhGMGZQcTE3U2lRaXUvMjdJR3pERVRlMFRuajE2TmhUVFZGUmFyMzEySHhrTzh4TDVzMgo0ZTBFUW9EdDJ1cFgya3RsbUt2bmZoTU41L09ZYnoxYm5NdmlIazRtT0FuZDkzRDRQSHZhcFAzU1ZlbG9lY2U1Cm56RGtpU2tQNXpsTGMvb21vbUluaS9MQ1VvV3hhNzhKSFFFVlp4VTFoNnJUV2krb0RNTzA0VzV5R00ySGZ6RUQKdlNkUGxqNGk2VG1TY3NsY3QrVXg4Mmt0V2ZNWjVOMHpVREhrVHdJREFRQUJBb0lCQUNxOFZnNUloamwwbzladwpqcENKaW5BUVQybWtFUDE0L3pzcFE2RkhVdjc4MmEwWGVxeFFzeEQ1d1h3TmFjMnVraGZ2TXFuQ0Z2anZoak5kCjVVcGQwTGp4NUpjY3dYT3d5VVh5ZmRlRUNjd0ltbzk2STFPNGVXVFUySThuVjI2c0t0dDNEbkF6RTFXTjFlWUoKclJjMHJMZ2JTUU5sNFhZZlYvTXkxcHRBb0JiMklrUEg1UlM1bEt2MFVaZFJKclFla0hNK1VmNXJoS2ZSWTJmSgowNzJHcXBvMDBtOThtOHlDbTJVOWZXemg0QXU5ZVZ1NjJhcGNIZlM5M2xQaUhZdUNSY2tXMFFDeTRoRC9ONjA4CnR5Z2YyV0Y5OGVZQkdmUFM2N3FOVFFqM3FHVk9JWDBIVGdDT3dhYXR5OVBhc0k5WHh6WDdWQjlabjJjOU9SdEsKczM3LyttRUNnWUVBN1RlQkdSQU5sV2E1blQ0b3lpMVY2elV0b0V1cEluK0orTVhQMWwwZjdmdVQ3UU5TaXFIdApQVjJIVFg4eDU5dTRobkFnM2xVdEpHL1Y0NG1YZEZGQTJBNDVVazFrTlZLaVk4dXNaVG1FYkswWFhpa0lYYzN2CnRiYzBqTkpjTVlZWEp2UDlENmJKcEFENkxod0QxRVdTU2tYejcwd25sYjNFK3hOQ0I4TWVUMWtDZ1lFQXdMQ3UKalQ2ZTBJYlU1cG9JNThhSFJqTVRJYTVLN1JWWGRKOVNYd21EckUvcFVEWmxlWUZZSDZyaERkZmxHV3UwMUxOaApIUktnak8vb0tQbWxtbmFENHNjQThkaHRKYUQ3RmxNaDhUTlBNdWVpVTdzbmdMcDh1d3k0cGo0eFp0aEN5SzVQCnJsNk44VGJ4ZHh2RWNORmtSU3dsNkd4NWhwOTNkVVoyWjJzMFErY0NnWUI2UnVBZkFTMWZVOEplRVlxSkhQRnEKK3RCNERrZ3k1amRDcXdURlpOOXVsdjNiY0pqOXFSWlhTWHpUTDd1VDNxaXhjOThkZkI1MjU4Y0RNUXVIQTlNLwpDNlgwakx3WTJHMm9yM1kvLzNRSmFQZmdxNW9LY0hzZDJrQVdUdUVERGpHay9LUDNpMnZwUkMreDBQTVFXb1JjCkVNNzdEUEJpdVoxdUpyRWt1eWxHa1FLQmdRQ0dGaXpSUXlNMnpxeTJiN2dFdVlVRzE2SFhqVGE4ZGw1dVhEUHcKeUdrbGJsSllMQkkvRkQvamdZQ3NwbkRaV0xiMFVJTWl2UVNXTHBmcWM1Ykd2dlFWeWcraE84N3dJVzY0WVU4OApUazB0aWp6T2NXMFN5akxqNHYxWFNlNE11Qzc1QVR1WGhsclY0VHIwZkpFZFJNaTJ6ZnJ2R3hVU0ZrUUZpdXY0CkdZUzhQd0tCZ0QzQlRneGJxNXVHWmlvMFRkbFlhZW8vL1c5Y3F0akoyS0JZTm1Jb2N2VWV6T2ljeTlZRVAzVVEKNnVOU2YxRXZWN3BoNFlGNzJNN0ExYUZzTklhSnYyb3IwVWZWVWVKT3k0ejFkaUxHQklJcFpnT0JySmREcXhVKwphek1ZUW9CK0lnbmVWWDZ3YkRzaTBGU00xaUh5QTFka1lHQkpxTUFNaTFwR012SDlaTlpNCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0tCg==
+