Remove custom rbac for flannel (#314)

kubernetes-sigs · Jul 12, 2023 · 3ed2829 · 3ed2829 · FangKee · Jul 13, 2023
1 parent 6eb6a72
commit 3ed2829
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 95 deletions.
diff --git a/guides/flannel.md b/guides/flannel.md
@@ -59,40 +59,43 @@ The output should include the Linux flannel DaemonSet as running:
 ```
 NAMESPACE NAME READY STATUS RESTARTS AGE
 ...
-kube-system kube-flannel-ds-54954 1/1 Running 0 1m
+kube-flannel kube-flannel-ds-sfqkv 1/1 Running 0 1m
 ```
 
 ### Add Windows flannel solution
 
-1. Add Windows Flannel and kube-proxy DaemonSets
-
-Now you can add Windows-compatible versions of Flannel and kube-proxy. In order to ensure that you get a compatible version of kube-proxy, you'll need to substitute the tag of the image. The following example shows usage for Kubernetes v1.27.1, but you should adjust the version for your own deployment.
+Now you can add a Windows-compatible version of Flannel.
 
 ```bash
-curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/flannel-overlay.yml | sed 's/FLANNEL_VERSION/v0.21.5/g' | kubectl apply -f -
-curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/kube-proxy/kube-proxy.yml | sed 's/KUBE_PROXY_VERSION/v1.27.1/g' | kubectl apply -f -
+controlPlaneEndpoint=$(kubectl get configmap -n kube-system kube-proxy -o jsonpath="{.data['kubeconfig\.conf']}" | grep server: | sed 's/.*\:\/\///g')
+kubernetesServiceHost=$(echo $controlPlaneEndpoint | cut -d ":" -f 1)
+kubernetesServicePort=$(echo $controlPlaneEndpoint | cut -d ":" -f 2)
+curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/flannel-overlay.yml | sed 's/FLANNEL_VERSION/v0.21.5/g' | sed "s/KUBERNETES_SERVICE_HOST_VALUE/$kubernetesServiceHost/g" | sed "s/KUBERNETES_SERVICE_PORT_VALUE/$kubernetesServicePort/g" | kubectl apply -f -
 ```
 
->  **Note** If you are using another version of kubernetes on your Windows node, change v1.27.1 with your own version .
-> To find your version of kubernetes run the following command:
-> `kubeadm version`
+>  **Note** If your cluster uses a different service subnet than `10.96.0.0/12` then you need to adjust the environment variable `SERVICE_SUBNET` before applying it.
+> To find your service subnet run the following command:  
+> `kubectl get configmap -n kube-system kubeadm-config -o yaml | grep serviceSubnet`
 
 >  **Note** If you changed the`$CNIBinPath` or `$CNIConfigPath` optional parameters when running `Install-Containerd.ps1`,
->  you will need to use those paths on `flannel-overlay.yml`. Pipe it through
+>  you will need to use those paths. Pipe it through
 >  `| sed 's/C:\\\\opt\\\\cni\\\\bin/<your cni bin path>/g' | sed 's/C:\\\\etc\\\\cni\\\\net.d/<your cni config path>/g'`
 >  before feeding it to `kubectl apply -f -`.
 
->  **Note** If you changed the`$CNIBinPath` optional parameter when running `Install-Containerd.ps1`, you will need to
->  use that path on `kube-proxy.yml`. Pipe it through `| sed 's/C:\\\\opt\\\\cni\\\\bin/<your cni bin path>/g'` before
->  feeding it to `kubectl apply -f -`.
-
-2. Apply kube-flannel-rbac.yml from sig-windows-tools/kubeadm/flannel
-Next you will need to apply the configuration that allows flannel to spawn pods and keep them running:
+Next add a Windows-compatible version of kube-proxy. In order to ensure that you get a compatible version of kube-proxy, you'll need to substitute the tag of the image. The following example shows usage for Kubernetes v1.27.1, but you should adjust the version for your own deployment.
 
 ```bash
-kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/flanneld/kube-flannel-rbac.yml
+curl -L https://raw.githubusercontent.com/kubernetes-sigs/sig-windows-tools/master/hostprocess/flannel/kube-proxy/kube-proxy.yml | sed 's/KUBE_PROXY_VERSION/v1.27.1/g' | kubectl apply -f -
 ```
 
+>  **Note** If you are using another version of kubernetes on your Windows node, change v1.27.1 with your own version .
+> To find your version of kubernetes run the following command:
+> `kubeadm version`
+
+>  **Note** If you changed the`$CNIBinPath` optional parameter when running `Install-Containerd.ps1`, you will need to
+>  use that path. Pipe it through `| sed 's/C:\\\\opt\\\\cni\\\\bin/<your cni bin path>/g'` before
+>  feeding it to `kubectl apply -f -`.
+
 ## Verifying your installation for Flannel
 
 You should now be able to view the Windows node in your cluster by running:
@@ -101,10 +104,10 @@ You should now be able to view the Windows node in your cluster by running:
 kubectl get nodes -o wide
 ```
 
-If your new node is in the `NotReady` state it is likely because the flannel image is still downloading. You can check the progress as before by checking on the flannel pods in the `kube-system` namespace:
+If your new node is in the `NotReady` state it is likely because the flannel image is still downloading. You can check the progress as before by checking on the flannel pods in the `kube-flannel` namespace:
 
 ```shell
-kubectl -n kube-system get pods -l app=flannel
+kubectl -n kube-flannel get pods -l app=flannel
 ```
 
 Once the flannel Pod is running, your node should enter the `Ready` state and then be available to handle workloads.
diff --git a/hostprocess/flannel/flanneld/flannel-overlay.yml b/hostprocess/flannel/flanneld/flannel-overlay.yml
@@ -3,7 +3,7 @@ kind: ConfigMap
 apiVersion: v1
 metadata:
   name: kube-flannel-windows-cfg
-  namespace: kube-system
+  namespace: kube-flannel
   labels:
     tier: node
     app: flannel
@@ -59,7 +59,7 @@ metadata:
   labels:
     tier: node
     app: flannel
-  namespace: kube-system
+  namespace: kube-flannel
 spec:
   selector:
     matchLabels:
@@ -106,15 +106,23 @@ spec:
           mountPath: /mounts/kube-flannel/
         - name: flannel-windows-cfg
           mountPath: /mounts/kube-flannel-windows/
-        - name: kubeadm-config
-          mountPath: /mounts/kubeadm-config/
-        - name: kube-proxy
-          mountPath: /flannel-config-file
         env:
         - name: CNI_BIN_PATH
           value: C:\\opt\\cni\\bin
         - name: CNI_CONFIG_PATH
           value: C:\\etc\\cni\\net.d
+        - name: SERVICE_SUBNET
+          value: 10.96.0.0/12
+          # As of now with the currently used flannel version (last checked with v0.21.5) we need to overwrite KUBERNETES_SERVICE_HOST and KUBERNETES_SERVICE_PORT
+          # in order to be able to reach the kubernetes api server. Under windows it's currently not possible to reach it over the service created by kubernetes
+          # For more context and details check the corresponding PR: https://github.com/kubernetes-sigs/sig-windows-tools/pull/314
+          # Especially the comments in this review: https://github.com/kubernetes-sigs/sig-windows-tools/pull/314#discussion_r1238815189
+          # There is also a follow up issue on the flannel side: https://github.com/flannel-io/flannel/issues/1772
+          # Once this issue is solved we should be able to remove the custom host and port to the kubernetes api server
+        - name: KUBERNETES_SERVICE_HOST
+          value: KUBERNETES_SERVICE_HOST_VALUE
+        - name: KUBERNETES_SERVICE_PORT
+          value: "KUBERNETES_SERVICE_PORT_VALUE"
         - name: POD_NAME
           valueFrom:
             fieldRef:
@@ -126,15 +134,9 @@ spec:
               apiVersion: v1
               fieldPath: metadata.namespace
       volumes:
-      - name: kube-proxy
-        configMap:
-          name: kube-proxy
       - name: flannel-cfg
         configMap:
           name: kube-flannel-cfg
       - name: flannel-windows-cfg
         configMap:
           name: kube-flannel-windows-cfg
-      - name: kubeadm-config
-        configMap:
-          name: kubeadm-config
diff --git a/hostprocess/flannel/flanneld/kube-flannel-rbac.yml b/hostprocess/flannel/flanneld/kube-flannel-rbac.yml
diff --git a/hostprocess/flannel/flanneld/start.ps1 b/hostprocess/flannel/flanneld/start.ps1
@@ -15,8 +15,8 @@ cp -force $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel/net-conf.json
 # get info
 Write-Host "update cni config"
 $cniJson = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel-windows/cni-conf-containerd.json | ConvertFrom-Json
-$serviceSubnet = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kubeadm-config/ClusterConfiguration | ForEach-Object -Process {if($_.Contains("serviceSubnet:")) {$_.Trim().Split()[1]}}
-$podSubnet = get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kubeadm-config/ClusterConfiguration | ForEach-Object -Process {if($_.Contains("podSubnet:")) {$_.Trim().Split()[1]}}
+$serviceSubnet = $env:SERVICE_SUBNET
+$podSubnet = (get-content $env:CONTAINER_SANDBOX_MOUNT_POINT/mounts/kube-flannel/net-conf.json | ConvertFrom-Json).Network
 $na = Get-NetRoute | Where { $_.DestinationPrefix -eq '0.0.0.0/0' } | Select-Object -Property ifIndex
 $managementIP = (Get-NetIPAddress -ifIndex $na[0].ifIndex -AddressFamily IPv4).IPAddress
 
@@ -32,13 +32,9 @@ Set-Content -Path $env:CNI_CONFIG_PATH/10-flannel.conf ($cniJson | ConvertTo-Jso
 Write-Host "add route"
 route /p add 169.254.169.254 mask 255.255.255.255 0.0.0.0
 
-write-host "copy sa info (should be able to do this with a change to go client"
-mkdir -force $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/var/run/secrets/kubernetes.io/serviceaccount/
-cp -force $env:CONTAINER_SANDBOX_MOUNT_POINT/var/run/secrets/kubernetes.io/serviceaccount/* $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/var/run/secrets/kubernetes.io/serviceaccount/
-
 Write-Host "envs"
 write-host $env:POD_NAME
 write-host $env:POD_NAMESPACE
 
 Write-Host "Starting flannel"
-& $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel/flanneld.exe --kube-subnet-mgr --kubeconfig-file $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel-config-file/kubeconfig.conf --iface $managementIP
+& $env:CONTAINER_SANDBOX_MOUNT_POINT/flannel/flanneld.exe --kube-subnet-mgr --iface $managementIP