From f1efbcc9fe9712670e4f99f7d07173bb813ef83f Mon Sep 17 00:00:00 2001 From: jay vyas Date: Thu, 1 Jul 2021 12:02:44 -0400 Subject: [PATCH] Remove nat network (#42) * Antrea retries if kube proxy not online yet * remove that containerd nat CNI * fix containerd --- Vagrantfile | 6 +- forked/1-antrea.ps1 | 3 - forked/Install-Containerd.ps1 | 136 ++++++++++++++++++++++++++++++++++ forked/PrepareNode.ps1 | 2 +- sync/shared/kubejoin.ps1 | 2 +- sync/windows/containerd1.ps1 | 9 +-- sync/windows/k.ps1 | 3 +- 7 files changed, 145 insertions(+), 16 deletions(-) create mode 100644 forked/Install-Containerd.ps1 diff --git a/Vagrantfile b/Vagrantfile index 0001973..cc6934d 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -49,14 +49,16 @@ Vagrant.configure(2) do |config| winw1.vm.network :private_network, ip:"10.20.30.11" winw1.vm.synced_folder ".", "/vagrant", disabled:true winw1.vm.synced_folder "./sync/shared", "C:/sync/shared" - winw1.vm.synced_folder "./sync/windows/bin/", "C:/sync/windows/bin" + winw1.vm.synced_folder "./sync/windows/", "C:/sync/windows/" + winw1.vm.synced_folder "./forked", "C:/forked/" + winw1.vm.provider :virtualbox do |vb| vb.memory = windows_ram vb.cpus = windows_cpus vb.gui = false end - winw1.vm.provision "shell", path: "sync/windows/hyperv.ps1", privileged: true #, run: "never" + winw1.vm.provision "shell", path: "sync/windows/hyperv.ps1", privileged: true winw1.vm.provision :reload winw1.vm.provision "shell", path: "sync/windows/containerd1.ps1", privileged: true #, run: "never" winw1.vm.provision :reload diff --git a/forked/1-antrea.ps1 b/forked/1-antrea.ps1 index 3f3042a..327df9f 100644 --- a/forked/1-antrea.ps1 +++ b/forked/1-antrea.ps1 @@ -131,9 +131,6 @@ Get-Service *kube* Get-Service *antrea* Get-Service *ovs* -################################################## -# Try starting antrea. Restart it, just in case # -################################################## $antrea = Get-Service -Name "antrea-agent" $antrea_starts = 0 while ($antrea.Status -ne 'Running') diff --git a/forked/Install-Containerd.ps1 b/forked/Install-Containerd.ps1 new file mode 100644 index 0000000..1ac56d9 --- /dev/null +++ b/forked/Install-Containerd.ps1 @@ -0,0 +1,136 @@ +<# +.SYNOPSIS +Installs ContainerD on a Windows machines in preperation for joining the node to a Kubernetes cluster. + +.DESCRIPTION +This script +- Verifies that Windows Features requried for running contianers are enabled (and enables then if they are not) +- Downloads ContainerD binaries from from at the version specified. +- Downloads Windows SND CNI plugins. +- Sets up a basic nat networking config for ContainerD to use until another CNI is configured +- Registers ContainerD as a windows service. + +.PARAMETER ContainerDVersion +ContainerD version to download and use. + +.PARAMETER netAdapterName +Name of network adapter to use when configuring basic nat network. + +.EXAMPLE +PS> .\Install-Conatinerd.ps1 + +#> + +Param( + [parameter(HelpMessage = "ContainerD version to use")] + [string] $ContainerDVersion = "1.4.1", + [parameter(HelpMessage = "Name of network adapter to use when configuring basic nat network")] + [string] $netAdapterName = "Ethernet" +) + +$ErrorActionPreference = 'Stop' + +function DownloadFile($destination, $source) { + Write-Host("Downloading $source to $destination") + curl.exe --silent --fail -Lo $destination $source + + if (!$?) { + Write-Error "Download $source failed" + exit 1 + } +} + +<# +.DESCRIPTION +Computes a subnet for a gateway from the IPv4 IPAddress and PrefixLength properties +for a given network adapter. This value is used for IPAM in a nat CNI config required for +containerd. + +.NOTES +This logic is adapted from +https://github.com/containerd/containerd/blob/4a6b47d470d9f2dfc3d49f2819b968861dfa123e/script/setup/install-cni-windows + +.EXAMPLE +PS> CalculateSubNet -gateway 172.16.5.8 -prefixLength 24 +172.16.5.0/8 +#> +function CalculateSubNet { + param ( + [string]$gateway, + [int]$prefixLength + ) + $len = $prefixLength + $parts = $gateway.Split('.') + $result = @() + for ($i = 0; $i -le 3; $i++) { + if ($len -ge 8) { + $mask = 255 + + } + elseif ($len -gt 0) { + $mask = ((256 - 2 * (8 - $len))) + } + else { + $mask = 0 + } + $len -= 8 + $result += ([int]$parts[$i] -band $mask) + } + + $subnetIp = [string]::Join('.', $result) + $cidr = 32 - $prefixLength + return "${subnetIp}/$cidr" +} + +$requiredWindowsFeatures = @( + "Containers", + "Hyper-V", + "Hyper-V-PowerShell") + +function ValidateWindowsFeatures { + $allFeaturesInstalled = $true + foreach ($feature in $requiredWindowsFeatures) { + $f = Get-WindowsFeature -Name $feature + if (-not $f.Installed) { + Write-Warning "Windows feature: '$feature' is not installed." + $allFeaturesInstalled = $false + } + } + return $allFeaturesInstalled +} + +if (-not (ValidateWindowsFeatures)) { + Write-Output "Installing required windows features..." + + foreach ($feature in $requiredWindowsFeatures) { + Install-WindowsFeature -Name $feature + } + + Write-Output "Please reboot and re-run this script." + exit 0 +} + +Write-Output "Getting ContainerD binaries" +$global:ConainterDPath = "$env:ProgramFiles\containerd" +mkdir -Force $global:ConainterDPath | Out-Null +DownloadFile "$global:ConainterDPath\containerd.tar.gz" https://github.com/containerd/containerd/releases/download/v${ContainerDVersion}/containerd-${ContainerDVersion}-windows-amd64.tar.gz +tar.exe -xvf "$global:ConainterDPath\containerd.tar.gz" --strip=1 -C $global:ConainterDPath +$env:Path += ";$global:ConainterDPath" +[Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) +containerd.exe config default | Out-File "$global:ConainterDPath\config.toml" -Encoding ascii +#config file fixups +$config = Get-Content "$global:ConainterDPath\config.toml" +$config = $config -replace "bin_dir = (.)*$", "bin_dir = `"c:/opt/cni/bin`"" +$config = $config -replace "conf_dir = (.)*$", "conf_dir = `"c:/etc/cni/net.d`"" +$config | Set-Content "$global:ConainterDPath\config.toml" -Force + +mkdir -Force c:\opt\cni\bin | Out-Null +mkdir -Force c:\etc\cni\net.d | Out-Null + +Write-Output "Registering ContainerD as a service" +containerd.exe --register-service + +Write-Output "Starting ContainerD service" +Start-Service containerd + +Write-Output "Done - please remember to add '--cri-socket `"npipe:////./pipe/containerd-containerd`"' to your kubeadm join command" \ No newline at end of file diff --git a/forked/PrepareNode.ps1 b/forked/PrepareNode.ps1 index 963bdd4..fece6b3 100644 --- a/forked/PrepareNode.ps1 +++ b/forked/PrepareNode.ps1 @@ -132,7 +132,7 @@ DownloadFile "$global:KubernetesPath\kubeadm.exe" https://dl.k8s.io/$KubernetesV if ($ContainerRuntime -eq "Docker") { # Create host network to allow kubelet to schedule hostNetwork pods # NOTE: For containerd the 0-containerd-nat.json network config template added by - # Install-containerd.ps1 joins pods to the host network. + # Install-containerd.ps1 joins pods to the host network. but it doesnt work . Write-Host "Creating Docker host network" docker network create -d nat host } elseif ($ContainerRuntime -eq "containerD") { diff --git a/sync/shared/kubejoin.ps1 b/sync/shared/kubejoin.ps1 index 1fa0b7b..5ce663f 100644 --- a/sync/shared/kubejoin.ps1 +++ b/sync/shared/kubejoin.ps1 @@ -1,3 +1,3 @@ $env:path += ";C:\Program Files\containerd" [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine) -kubeadm join 10.20.30.10:6443 --cri-socket "npipe:////./pipe/containerd-containerd" --token 86039w.99bp7lykobg831qx --discovery-token-ca-cert-hash sha256:6f4cac90bb19a1af3c620eb4bbd015d00b2181653ab6f36a3bf5ebce0dc01e76 +kubeadm join 10.20.30.10:6443 --cri-socket "npipe:////./pipe/containerd-containerd" --token fewn1r.evy8krm0f4xvqcac --discovery-token-ca-cert-hash sha256:f87ed7d225085d86c5b93b7dce2dc20d38aacea803d7af3f158ddf7804720dec diff --git a/sync/windows/containerd1.ps1 b/sync/windows/containerd1.ps1 index e0e0350..20876aa 100644 --- a/sync/windows/containerd1.ps1 +++ b/sync/windows/containerd1.ps1 @@ -25,17 +25,12 @@ $ProgressPreference = 'SilentlyContinue' #Write-Output "### Enabling Hyper-V-PowerShell-Module" #Enable-WindowsOptionalFeature -Online -FeatureName Microsoft-Hyper-V-Management-PowerShell - Set-Location 'C:\k' -Write-Output "#Curling 'Install-Containerd.ps1'" - -curl.exe -LO 'https://github.com/kubernetes-sigs/sig-windows-tools/releases/latest/download/Install-Containerd.ps1' - Write-Output "# Running 'Install-Containerd.ps1'" -PowerShell "C:\k\Install-Containerd.ps1" - +# Our own version of install-containerd that omits the weird nat cni network thing +PowerShell "C:/forked/Install-Containerd.ps1" # To avoid the "crictl.exe not on the path error, we add containerd permanantly to the pathhhhh" # TODO THIS might not be needed ... diff --git a/sync/windows/k.ps1 b/sync/windows/k.ps1 index 6dc129d..3de90c8 100644 --- a/sync/windows/k.ps1 +++ b/sync/windows/k.ps1 @@ -21,8 +21,7 @@ $ErrorActionPreference = 'Stop' Write-Output "Kubernetes Version $KubernetesVersion" dism /online /get-features -curl.exe -LO https://github.com/kubernetes-sigs/sig-windows-tools/releases/latest/download/Install-Containerd.ps1 -.\Install-Containerd.ps1 +PowerShell C:/forked/Install-Containerd.ps1 ctr.exe version New-Item -ItemType Directory -Force -Path C:\k