Skip to content

Commit

Permalink
Bump suc to 103.0.2+up0.6.1, backport windows system agent upgrader c…
Browse files Browse the repository at this point in the history 
…hanges
  • Loading branch information
@HarrisonWAffel
HarrisonWAffel committed Sep 4, 2024
1 parent 429640c commit 69e442e
Show file tree
Hide file tree
Showing 8 changed files with 267 additions and 109 deletions.
4 changes: 2 additions & 2 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ require (
github.com/Azure/go-autorest/autorest/adal v0.9.23
github.com/Azure/go-autorest/autorest/to v0.4.1-0.20210111195520-9fc88b15294e
github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1
github.com/Masterminds/semver/v3 v3.2.1
github.com/Masterminds/semver/v3 v3.3.0
github.com/Masterminds/sprig/v3 v3.2.3
github.com/aws/aws-sdk-go v1.49.24
github.com/bep/debounce v1.2.0
Expand Down Expand Up @@ -125,7 +125,7 @@ require (
github.com/rancher/remotedialer v0.4.0
github.com/rancher/rke v1.5.12
github.com/rancher/steve v0.0.0-20240529152548-9fb3e50aa806
github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20210727200656-10b094e30007
github.com/rancher/system-upgrade-controller/pkg/apis v0.0.0-20240301001845-4eacc2dabbde
github.com/rancher/wrangler/v2 v2.1.4
github.com/robfig/cron v1.2.0
github.com/sirupsen/logrus v1.9.3
Expand Down
41 changes: 4 additions & 37 deletions go.sum
View file

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion package/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ ENV CATTLE_SYSTEM_AGENT_INSTALLER_IMAGE rancher/system-agent-installer-
# make sure the ENV CATTLE_SYSTEM_AGENT_INSTALL_SCRIPT is consistent with pkg/settings/setting.go to utlize the local version of install script downloaded during build/package
ENV CATTLE_SYSTEM_AGENT_INSTALL_SCRIPT ${CATTLE_SYSTEM_AGENT_DOWNLOAD_PREFIX}/${CATTLE_SYSTEM_AGENT_VERSION}/install.sh
ENV CATTLE_SYSTEM_AGENT_UNINSTALL_SCRIPT ${CATTLE_SYSTEM_AGENT_DOWNLOAD_PREFIX}/${CATTLE_SYSTEM_AGENT_VERSION}/system-agent-uninstall.sh
ENV CATTLE_SYSTEM_UPGRADE_CONTROLLER_CHART_VERSION 103.0.1+up0.6.0
ENV CATTLE_SYSTEM_UPGRADE_CONTROLLER_CHART_VERSION 103.0.2+up0.6.1

# System charts minimal version
# Deprecated in favor of CATTLE_FLEET_VERSION.
Expand Down
8 changes: 7 additions & 1 deletion pkg/capr/installer/installer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,11 @@ func WindowsInstallScript(ctx context.Context, token string, envVars []corev1.En
server = fmt.Sprintf("$env:CATTLE_SERVER=\"%s\"", settings.ServerURL.Get())
}

strictVerify := "false"
if settings.AgentTLSMode.Get() == settings.AgentTLSModeStrict {
strictVerify = "true"
}

return []byte(fmt.Sprintf(`%s
%s
Expand All @@ -186,8 +191,9 @@ func WindowsInstallScript(ctx context.Context, token string, envVars []corev1.En
$env:CSI_PROXY_URL = "%s"
$env:CSI_PROXY_VERSION = "%s"
$env:CSI_PROXY_KUBELET_PATH = "C:/var/lib/rancher/rke2/bin/kubelet.exe"
$env:STRICT_VERIFY = "%s"
Invoke-WinsInstaller @PSBoundParameters
exit 0
`, data, envVarBuf.String(), binaryURL, server, ca, token, csiProxyURL, csiProxyVersion)), nil
`, data, envVarBuf.String(), binaryURL, server, ca, token, csiProxyURL, csiProxyVersion, strictVerify)), nil
}
226 changes: 159 additions & 67 deletions pkg/controllers/capr/managesystemagent/managesystemagent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"github.com/rancher/wrangler/v2/pkg/generic"
"github.com/rancher/wrangler/v2/pkg/gvk"
"github.com/rancher/wrangler/v2/pkg/name"
"github.com/sirupsen/logrus"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
"k8s.io/apimachinery/pkg/api/meta"
Expand All @@ -35,24 +36,43 @@ import (
)

const (
generationSecretName = "system-agent-upgrade-generation"
generationSecretName = "system-agent-upgrade-generation"
upgradeAPIVersion = "upgrade.cattle.io/v1"
upgradeDigestAnnotation = "upgrade.cattle.io/digest"
systemAgentUpgraderServiceAccountName = "system-agent-upgrader"
)

var Kubernetes125 = semver.MustParse("v1.25.0")
var (
Kubernetes125 = semver.MustParse("v1.25.0")

// GH5551FixedVersions is a slice of rke2 versions
// which have resolved GH-5551 for Windows nodes.
// ref: https://github.com/rancher/rke2/issues/5551
// The SUC should not deploy plans to Windows nodes
// running a version less than the below for each minor.
// This check can be removed when 1.31.x is the lowest supported
// rke2 version.
GH5551FixedVersions = map[int]*semver.Version{
30: semver.MustParse("v1.30.4"),
29: semver.MustParse("v1.29.8"),
28: semver.MustParse("v1.28.13"),
27: semver.MustParse("v1.27.16"),
}
)

type handler struct {
clusterRegistrationTokens v3.ClusterRegistrationTokenCache
bundles fleetcontrollers.BundleClient
rkeControlPlane v1.RKEControlPlaneController
provClusters rocontrollers.ClusterCache
controlPlanes v1.RKEControlPlaneCache
}

func Register(ctx context.Context, clients *wrangler.Context) {
h := &handler{
clusterRegistrationTokens: clients.Mgmt.ClusterRegistrationToken().Cache(),
bundles: clients.Fleet.Bundle(),
rkeControlPlane: clients.RKE.RKEControlPlane(),
provClusters: clients.Provisioning.Cluster().Cache(),
controlPlanes: clients.RKE.RKEControlPlane().Cache(),
}

v1.RegisterRKEControlPlaneStatusHandler(ctx, clients.RKE.RKEControlPlane(),
Expand Down Expand Up @@ -125,6 +145,19 @@ func (h *handler) OnChange(cluster *rancherv1.Cluster, status rancherv1.ClusterS
})
}

cp, err := h.controlPlanes.Get(cluster.Namespace, cluster.Name)
if err != nil {
logrus.Errorf("Error encountered getting RKE control plane while determining SUC readiness: %v", err)
return nil, status, err
}

if !capr.SystemUpgradeControllerReady.IsTrue(cp) {
// If the SUC is not ready do not create any plans, as those
// plans may depend on functionality only a newer version of the SUC contains
logrus.Debugf("[managesystemagent] the SUC is not yet ready, waiting to create system agent upgrade plans (SUC status: %s)", capr.SystemUpgradeControllerReady.GetStatus(cp))
return nil, status, generic.ErrSkip
}

resources, err := ToResources(installer(cluster, secretName))
if err != nil {
return nil, status, err
Expand All @@ -138,6 +171,11 @@ func (h *handler) OnChange(cluster *rancherv1.Cluster, status rancherv1.ClusterS
Spec: v1alpha1.BundleSpec{
BundleDeploymentOptions: v1alpha1.BundleDeploymentOptions{
DefaultNamespace: namespaces.System,
// In the event that a controller updates the SUC Plan at the same time as
// fleet is attempting to update the plan via the bundle, we may end up with drift.
CorrectDrift: &v1alpha1.CorrectDrift{
Enabled: true,
},
},
Resources: resources,
Targets: []v1alpha1.BundleTarget{
Expand Down Expand Up @@ -166,12 +204,6 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
version = upgradeImage[1]
}

winsUpgradeImage := strings.SplitN(settings.WinsAgentUpgradeImage.Get(), ":", 2)
winsVersion := "latest"
if len(winsUpgradeImage) == 2 {
winsVersion = winsUpgradeImage[1]
}

var env []corev1.EnvVar
for _, e := range cluster.Spec.AgentEnvVars {
env = append(env, corev1.EnvVar{
Expand Down Expand Up @@ -207,17 +239,18 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
Value: "true",
})
}
var plans []runtime.Object

plan := &upgradev1.Plan{
TypeMeta: metav1.TypeMeta{
Kind: "Plan",
APIVersion: "upgrade.cattle.io/v1",
APIVersion: upgradeAPIVersion,
},
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
Namespace: namespaces.System,
Annotations: map[string]string{
"upgrade.cattle.io/digest": "spec.upgrade.envs,spec.upgrade.envFrom",
upgradeDigestAnnotation: "spec.upgrade.envs,spec.upgrade.envFrom",
},
},
Spec: upgradev1.PlanSpec{
Expand All @@ -238,7 +271,7 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
},
},
},
ServiceAccountName: "system-agent-upgrader",
ServiceAccountName: systemAgentUpgraderServiceAccountName,
Upgrade: &upgradev1.ContainerSpec{
Image: image.ResolveWithCluster(upgradeImage[0], cluster),
Env: env,
Expand All @@ -252,62 +285,28 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
},
},
}
plans = append(plans, plan)

windowsPlan := &upgradev1.Plan{
TypeMeta: metav1.TypeMeta{
Kind: "Plan",
APIVersion: "upgrade.cattle.io/v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader-windows",
Namespace: namespaces.System,
Annotations: map[string]string{
"upgrade.cattle.io/digest": "spec.upgrade.envs,spec.upgrade.envFrom",
},
},
Spec: upgradev1.PlanSpec{
Concurrency: 10,
Version: winsVersion,
Tolerations: []corev1.Toleration{{
Operator: corev1.TolerationOpExists,
},
},
NodeSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: corev1.LabelOSStable,
Operator: metav1.LabelSelectorOpIn,
Values: []string{
"windows",
},
},
},
},
ServiceAccountName: "system-agent-upgrader",
Upgrade: &upgradev1.ContainerSpec{
Image: image.ResolveWithCluster(winsUpgradeImage[0], cluster),
Env: env,
EnvFrom: []corev1.EnvFromSource{{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
},
}},
},
},
if currentVersionResolvesGH5551(cluster.Spec.KubernetesVersion) {
windowsPlan := winsUpgradePlan(cluster, env, secretName)
if cluster.Spec.RedeploySystemAgentGeneration != 0 {
windowsPlan.Spec.Secrets = append(windowsPlan.Spec.Secrets, upgradev1.SecretSpec{
Name: generationSecretName,
})
}
plans = append(plans, windowsPlan)
}

objs := []runtime.Object{
&corev1.ServiceAccount{
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
Namespace: namespaces.System,
},
},
&rbacv1.ClusterRole{
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
},
Rules: []rbacv1.PolicyRule{{
Verbs: []string{"get"},
Expand All @@ -317,17 +316,17 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
},
&rbacv1.ClusterRoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
},
Subjects: []rbacv1.Subject{{
Kind: "ServiceAccount",
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
Namespace: namespaces.System,
}},
RoleRef: rbacv1.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "system-agent-upgrader",
Name: systemAgentUpgraderServiceAccountName,
},
},
}
Expand All @@ -337,10 +336,6 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
Name: generationSecretName,
})

windowsPlan.Spec.Secrets = append(windowsPlan.Spec.Secrets, upgradev1.SecretSpec{
Name: generationSecretName,
})

objs = append(objs, &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: generationSecretName,
Expand All @@ -353,7 +348,104 @@ func installer(cluster *rancherv1.Cluster, secretName string) []runtime.Object {
})
}

return append([]runtime.Object{plan, windowsPlan}, objs...)
return append(plans, objs...)
}

func winsUpgradePlan(cluster *rancherv1.Cluster, env []corev1.EnvVar, secretName string) *upgradev1.Plan {
winsUpgradeImage := strings.SplitN(settings.WinsAgentUpgradeImage.Get(), ":", 2)
winsVersion := "latest"
if len(winsUpgradeImage) == 2 {
winsVersion = winsUpgradeImage[1]
}

return &upgradev1.Plan{
TypeMeta: metav1.TypeMeta{
Kind: "Plan",
APIVersion: upgradeAPIVersion,
},
ObjectMeta: metav1.ObjectMeta{
Name: "system-agent-upgrader-windows",
Namespace: namespaces.System,
Annotations: map[string]string{
upgradeDigestAnnotation: "spec.upgrade.envs,spec.upgrade.envFrom",
},
},
Spec: upgradev1.PlanSpec{
Concurrency: 10,
Version: winsVersion,
Tolerations: []corev1.Toleration{
{
Operator: corev1.TolerationOpExists,
},
},
NodeSelector: &metav1.LabelSelector{
MatchExpressions: []metav1.LabelSelectorRequirement{
{
Key: corev1.LabelOSStable,
Operator: metav1.LabelSelectorOpIn,
Values: []string{
"windows",
},
},
},
},
ServiceAccountName: systemAgentUpgraderServiceAccountName,
Upgrade: &upgradev1.ContainerSpec{
Image: image.ResolveWithCluster(winsUpgradeImage[0], cluster),
Env: env,
SecurityContext: &corev1.SecurityContext{
WindowsOptions: &corev1.WindowsSecurityContextOptions{
HostProcess: toBoolPointer(true),
RunAsUserName: toStringPointer("NT AUTHORITY\\SYSTEM"),
},
},
EnvFrom: []corev1.EnvFromSource{{
SecretRef: &corev1.SecretEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
},
},
}},
},
},
}
}

func toBoolPointer(x bool) *bool {
return &x
}

func toStringPointer(x string) *string {
return &x
}

// currentVersionResolvesGH5551 determines if the given rke2 version
// has fixed the RKE2 bug outlined in GH-5551. Windows SUC plans cannot be delivered
// to clusters running versions containing this bug. This function can be removed
// when v1.31.x is the lowest supported version offered by Rancher.
func currentVersionResolvesGH5551(version string) bool {

// remove leading v and trailing distro identifier
v := strings.TrimPrefix(version, "v")
verSplit := strings.Split(v, "+")
if len(verSplit) != 2 {
return false
}

curSemVer, err := semver.NewVersion(verSplit[0])
if err != nil {
return false
}

minor := curSemVer.Minor()
if minor >= 31 {
return true
}
if minor <= 26 {
return false
}

return curSemVer.GreaterThanEqual(GH5551FixedVersions[int(minor)])
}

func ToResources(objs []runtime.Object) (result []v1alpha1.BundleResource, err error) {
Expand Down
Loading

0 comments on commit 69e442e

Please sign in to comment.