Redeploy Fleet agent when TLS mode setting changes

This commit takes care of watching the agent TLS mode setting in the
`fleet-controller` config map, and of redeploying the Fleet agent to
upstream and downstream clusters when that setting changes.
Note that this only works for downstream clusters registered through a
manager-initiated process [1].

Testing this is done by reusing existing agent TLS mode test cases, and
triggering new deployments of the Fleet agent by patching the
`fleet-controller` config map.
Requirements for this include a cluster registered in manager-initiated
mode, while existing multi-cluster end-to-end tests need a downstream
cluster registered in agent-initiated mode.
Therefore, this commit also adds a new downstream cluster to the
multi-cluster CI workflow, which is so far only used for agent TLS mode
tests.

[1]: https://fleet.rancher.io/cluster-registration#manager-initiated

.github/scripts/deploy-fleet.sh

@@ -24,6 +24,10 @@ else
   agentTag="dev"
 fi
 
+host=$(kubectl get node k3d-upstream-server-0 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-upstream")].cluster.certificate-authority-data}' | base64 -d )
+server="https://$host:6443"
+
 eventually helm upgrade --install fleet-crd charts/fleet-crd \
   --atomic \
   -n cattle-fleet-system \
@@ -37,6 +41,8 @@ eventually helm upgrade --install fleet charts/fleet \
   --set agentImage.repository="$agentRepo" \
   --set agentImage.tag="$agentTag" \
   --set agentImage.imagePullPolicy=IfNotPresent \
+  --set apiServerCA="$ca" \
+  --set apiServerURL="$server" \
   --set shards="{$shards}" \
   --set debug=true --set debugLevel=1
 

.github/workflows/e2e-ci.yml

@@ -63,15 +63,15 @@ jobs:
         # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
         with:
           k3d-version: ${{ env.SETUP_K3D_VERSION }}
-          cluster-name: "k3s-default"
+          cluster-name: "upstream"
           args: >-
             --agents 1
             --network "nw01"
             --image docker.io/rancher/k3s:${{matrix.k3s_version}}
       -
         name: Import Images Into k3d
         run: |
-          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev nginx-git:test
+          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev nginx-git:test -c upstream
       -
         name: Set Up Tmate Debug Session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}

.github/workflows/e2e-multicluster-ci.yml

@@ -58,7 +58,7 @@ jobs:
             --agents 1
             --network "nw01"
       -
-        name: Provision k3d Downstream Cluster
+        name: Provision k3d Downstream Cluster for agent-initiated registration
         uses: AbsaOSS/k3d-action@v2
         with:
           k3d-version: ${{ env.SETUP_K3D_VERSION }}
@@ -69,11 +69,24 @@ jobs:
             --api-port 6644
             --agents 1
             --network "nw01"
+      -
+        name: Provision k3d Downstream Cluster for manager-initiated registration
+        uses: AbsaOSS/k3d-action@v2
+        with:
+          k3d-version: ${{ env.SETUP_K3D_VERSION }}
+          cluster-name: "managed-downstream"
+          args: >-
+            -p "82:80@agent:0:direct"
+            -p "445:443@agent:0:direct"
+            --api-port 6645
+            --agents 1
+            --network "nw01"
       -
         name: Import Images Into k3d
         run: |
           ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev -c upstream
           ./.github/scripts/k3d-import-retry.sh rancher/fleet-agent:dev -c downstream
+          ./.github/scripts/k3d-import-retry.sh rancher/fleet-agent:dev -c managed-downstream
       -
         name: Set Up Tmate Debug Session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}
@@ -127,6 +140,61 @@ jobs:
           while [ $(kubectl -n fleet-default get cluster -o jsonpath='{.items[0].status.summary.ready}') -ne 1 ]; do
             sleep 1
           done
+
+      -
+        name: Deploy and Register Managed Downstream Fleet
+        run: |
+          kubectl config use-context k3d-managed-downstream
+          host=$(kubectl get node k3d-managed-downstream-server-0 -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}')
+          ca=$( kubectl config view --flatten -o jsonpath='{.clusters[?(@.name == "k3d-managed-downstream")].cluster.certificate-authority-data}' )
+          client_cert=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-managed-downstream")].user.client-certificate-data}' )
+          token=$( kubectl config view --flatten -o jsonpath='{.users[?(@.name == "admin@k3d-managed-downstream")].user.client-key-data}' )
+          server="https://$host:6443"
+
+          kubectl config use-context k3d-upstream
+
+          value=$(cat <<EOF
+          apiVersion: v1
+          kind: Config
+          current-context: default
+          clusters:
+          - cluster:
+              certificate-authority-data: $ca
+              server: $server
+            name: cluster
+          contexts:
+          - context:
+              cluster: cluster
+              user: user
+            name: default
+          preferences: {}
+          users:
+          - name: user
+            user:
+              client-certificate-data: $client_cert
+              client-key-data: $token
+          EOF
+          )
+
+          kubectl create ns fleet-default || true
+          kubectl delete secret -n fleet-default kbcfg-second || true
+          # Rancher sets a token value in the secret, but our docs don't mention it
+          # * https://github.com/rancher/rancher/blob/c24fb8b0869a0b445f55b3307c6ed4582e147747/pkg/provisioningv2/kubeconfig/manager.go#L362
+          # * https://fleet.rancher.io/0.5/manager-initiated#kubeconfig-secret-1
+          kubectl create secret generic -n fleet-default kbcfg-second --from-literal=token="$token" --from-literal=value="$value"
+
+          kubectl apply -n fleet-default -f - <<EOF
+          apiVersion: "fleet.cattle.io/v1alpha1"
+          kind: Cluster
+          metadata:
+            name: second
+            namespace: fleet-default
+            labels:
+              name: second
+          spec:
+            kubeConfigSecret: kbcfg-second
+          EOF
+
       -
         name: E2E tests
         env:
@@ -135,7 +203,14 @@ jobs:
         run: |
           kubectl config use-context k3d-upstream
           ginkgo --github-output e2e/multi-cluster
-
+      -
+        name: E2E tests with managed downstream agent
+        env:
+          FLEET_E2E_NS: fleet-local
+          FLEET_E2E_NS_DOWNSTREAM: fleet-default
+          FLEET_E2E_CLUSTER_DOWNSTREAM: k3d-managed-downstream
+        run: |
+          kubectl config use-context k3d-upstream
           ginkgo --github-output e2e/multi-cluster/installation
       -
         name: Acceptance Tests for Examples

.github/workflows/nightly-ci.yml

@@ -64,15 +64,15 @@ jobs:
         # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16
         with:
           k3d-version: ${{ env.SETUP_K3D_VERSION }}
-          cluster-name: "k3s-default"
+          cluster-name: "upstream"
           args: >-
             --agents 1
             --network "nw01"
             --image docker.io/rancher/k3s:${{matrix.k3s_version}}
       -
         name: Import Images Into k3d
         run: |
-          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev nginx-git:test
+          ./.github/scripts/k3d-import-retry.sh rancher/fleet:dev rancher/fleet-agent:dev nginx-git:test -c upstream
       -
         name: Set Up Tmate Debug Session
         if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }}

charts/fleet-crd/templates/crds.yaml

@@ -5173,6 +5173,20 @@ spec:
                     used to detect changes.'
                   nullable: true
                   type: string
+                agentTLSMode:
+                  description: 'AgentTLSMode supports two values: `system-store` and
+                    `strict`. If set to
+
+                    `system-store`, instructs the agent to trust CA bundles from the
+                    operating
+
+                    system''s store. If set to `strict`, then the agent shall only
+                    connect to a
+
+                    server which uses the exact CA configured when creating/updating
+                    the agent.'
+                  nullable: true
+                  type: string
                 agentTolerationsHash:
                   description: 'AgentTolerationsHash is a hash of the agent''s tolerations
 

e2e/multi-cluster/installation/agent_test.go

@@ -2,7 +2,6 @@ package installation_test
 
 import (
 	"fmt"
-	"os/exec"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -13,57 +12,30 @@ import (
 var (
 	agentMode string
 	kd        kubectl.Command
+	ku        kubectl.Command
 )
 
 var _ = Describe("Fleet installation with TLS agent modes", func() {
 	BeforeEach(func() {
 		kd = env.Kubectl.Context(env.Downstream)
+		ku = env.Kubectl.Context(env.Upstream)
 	})
 
 	JustBeforeEach(func() {
-		cmd := exec.Command(
-			"helm",
-			"--kube-context",
-			"k3d-downstream",
-			"uninstall",
-			"fleet-agent",
+		out, err := ku.Patch(
+			"configmap",
+			"fleet-controller",
 			"-n",
 			"cattle-fleet-system",
+			"--type=merge",
+			"-p",
+			fmt.Sprintf(
+				`{"data":{"config":"{\"apiServerURL\": \"https://google.com\", \"apiServerCA\": \"\", \"agentTLSMode\": \"%s\"}"}}`,
+				agentMode,
+			),
 		)
-		out, err := cmd.CombinedOutput()
 		Expect(err).ToNot(HaveOccurred(), string(out))
 
-		deleteOut, err := kd.Delete("ns", "cattle-fleet-system", "--now")
-		Expect(err).ToNot(HaveOccurred(), deleteOut)
-
-		go func() {
-			cmd := exec.Command(
-				"helm",
-				"--kube-context",
-				"k3d-downstream",
-				"-n",
-				"cattle-fleet-system",
-				"upgrade",
-				"--install",
-				"--create-namespace",
-				"--wait",
-				"fleet-agent",
-				"../../../charts/fleet-agent",
-				"--set-string",
-				"labels.env=test",
-				"--set",
-				`apiServerCA=`,
-				"--set",
-				`apiServerURL=https://google.com`,
-				"--set",
-				"clusterNamespace=fleet-default",
-				"--set",
-				"token=foo", // we don't need a correct token for this.
-				"--set",
-				fmt.Sprintf("agentTLSMode=%s", agentMode),
-			)
-			_ = cmd.Run()
-		}()
 	})
 
 	Context("with non-strict agent TLS mode", func() {

internal/cmd/controller/agentmanagement/controllers/cluster/import.go

@@ -92,7 +92,12 @@ func (i *importHandler) onConfig(config *config.Config) error {
 		if cluster.Spec.KubeConfigSecret == "" {
 			continue
 		}
-		if config.APIServerURL != cluster.Status.APIServerURL || hashStatusField(config.APIServerCA) != cluster.Status.APIServerCAHash {
+
+		hasConfigChanged := config.APIServerURL != cluster.Status.APIServerURL ||
+			hashStatusField(config.APIServerCA) != cluster.Status.APIServerCAHash ||
+			config.AgentTLSMode != cluster.Status.AgentTLSMode
+
+		if hasConfigChanged {
 			logrus.Infof("API server config changed, trigger cluster import for cluster %s/%s", cluster.Namespace, cluster.Name)
 			c := cluster.DeepCopy()
 			c.Status.AgentConfigChanged = true
@@ -388,6 +393,8 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
 	status.AgentConfigChanged = false
 	status.APIServerURL = apiServerURL
 	status.APIServerCAHash = hashStatusField(apiServerCA)
+	status.AgentTLSMode = cfg.AgentTLSMode
+
 	return status, nil
 }
 

pkg/apis/fleet.cattle.io/v1alpha1/cluster_types.go

@@ -190,6 +190,13 @@ type ClusterStatus struct {
 	// +nullable
 	APIServerCAHash string `json:"apiServerCAHash,omitempty"`
 
+	// AgentTLSMode supports two values: `system-store` and `strict`. If set to
+	// `system-store`, instructs the agent to trust CA bundles from the operating
+	// system's store. If set to `strict`, then the agent shall only connect to a
+	// server which uses the exact CA configured when creating/updating the agent.
+	// +nullable
+	AgentTLSMode string `json:"agentTLSMode,omitempty"`
+
 	// Display contains the number of ready bundles, nodes and a summary state.
 	Display ClusterDisplay `json:"display,omitempty"`
 	// AgentStatus contains information about the agent.