diff --git a/internal/kubecmd/BUILD.bazel b/internal/kubecmd/BUILD.bazel index 2e0f3ad222..757d1aa0c5 100644 --- a/internal/kubecmd/BUILD.bazel +++ b/internal/kubecmd/BUILD.bazel @@ -69,6 +69,7 @@ go_test( "@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured", "@io_k8s_apimachinery//pkg/runtime", "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3", "@io_k8s_sigs_yaml//:yaml", ], ) diff --git a/internal/kubecmd/kubecmd.go b/internal/kubecmd/kubecmd.go index 6630d3544f..76cc51ee6d 100644 --- a/internal/kubecmd/kubecmd.go +++ b/internal/kubecmd/kubecmd.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "slices" "sort" "strings" "time" @@ -128,6 +129,12 @@ func (k *KubeCmd) UpgradeNodeVersion(ctx context.Context, conf *config.Config, f case err != nil: return fmt.Errorf("updating image version: %w", err) } + + // TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released. + if err := k.reconcileKubeadmConfigMap(ctx); err != nil { + return fmt.Errorf("reconciling kubeadm config: %w", err) + } + k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.Version()) nodeVersion.Spec.ImageReference = imageReference nodeVersion.Spec.ImageVersion = imageVersion.Version() @@ -393,6 +400,44 @@ func (k *KubeCmd) applyNodeVersion(ctx context.Context, nodeVersion updatev1alph return updatedNodeVersion, err } +func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error { + clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) + if err != nil { + return fmt.Errorf("getting ClusterConfig: %w", err) + } + + for i, v := range clusterConfiguration.APIServer.ExtraVolumes { + if v.Name == "konnectivity-uds" { + clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) + } + } + for i, v := range clusterConfiguration.APIServer.ExtraVolumes { + if v.Name == "egress-config" { + clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) + } + } + delete(clusterConfiguration.APIServer.ExtraArgs, "egress-selector-config-file") + + newConfigYAML, err := yaml.Marshal(clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + if kubeadmConfig.Data[constants.ClusterConfigurationKey] == string(newConfigYAML) { + k.log.Debugf("No changes to kubeadm config required") + return nil + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debugf("Triggering kubeadm config update now") + if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + fmt.Fprintln(k.outWriter, "Successfully reconciled the cluster's kubeadm config") + return nil +} + // isValidImageUpdate checks if the new image version is a valid upgrade, and there is no upgrade already running. func (k *KubeCmd) isValidImageUpgrade(nodeVersion updatev1alpha1.NodeVersion, newImageVersion string, force bool) error { if !force { diff --git a/internal/kubecmd/kubecmd_test.go b/internal/kubecmd/kubecmd_test.go index c905e38cf4..0cc2af2cd8 100644 --- a/internal/kubecmd/kubecmd_test.go +++ b/internal/kubecmd/kubecmd_test.go @@ -11,6 +11,7 @@ import ( "encoding/json" "errors" "io" + "strings" "testing" "time" @@ -34,9 +35,61 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) func TestUpgradeNodeVersion(t *testing.T) { + clusterConf := kubeadmv1beta3.ClusterConfiguration{ + APIServer: kubeadmv1beta3.APIServer{ + ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ + ExtraArgs: map[string]string{}, + ExtraVolumes: []kubeadmv1beta3.HostPathMount{}, + }, + }, + } + + clusterConfBytes, err := json.Marshal(clusterConf) + require.NoError(t, err) + validKubeadmConfig := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeadmConfigMap, + }, + Data: map[string]string{ + constants.ClusterConfigurationKey: string(clusterConfBytes), + }, + } + + clusterConfWithKonnectivity := kubeadmv1beta3.ClusterConfiguration{ + APIServer: kubeadmv1beta3.APIServer{ + ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + "egress-selector-config-file": "/etc/kubernetes/egress-selector-config-file.yaml", + }, + ExtraVolumes: []kubeadmv1beta3.HostPathMount{ + { + Name: "egress-config", + HostPath: "/etc/kubernetes/egress-selector-config-file.yaml", + }, + { + Name: "konnectivity-uds", + HostPath: "/some/path/to/konnectivity-uds", + }, + }, + }, + }, + } + + clusterConfBytesWithKonnectivity, err := json.Marshal(clusterConfWithKonnectivity) + require.NoError(t, err) + validKubeadmConfigWithKonnectivity := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeadmConfigMap, + }, + Data: map[string]string{ + constants.ClusterConfigurationKey: string(clusterConfBytesWithKonnectivity), + }, + } + testCases := map[string]struct { kubectl *stubKubectl conditions []metav1.Condition @@ -63,7 +116,25 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantUpdate: true, + }, + "success with konnectivity migration": { + conf: func() *config.Config { + conf := config.Default() + conf.Image = "v1.2.3" + conf.KubernetesVersion = supportedValidK8sVersions()[1] + return conf + }(), + currentImageVersion: "v1.2.2", + currentClusterVersion: supportedValidK8sVersions()[0], + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfigWithKonnectivity, }, }, wantUpdate: true, @@ -79,7 +150,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -100,7 +172,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -119,8 +192,12 @@ func TestUpgradeNodeVersion(t *testing.T) { }(), currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - wantErr: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantErr: true, assertCorrectError: func(t *testing.T, err error) bool { var upgradeErr *compatibility.InvalidUpgradeError return assert.ErrorAs(t, err, &upgradeErr) @@ -139,8 +216,12 @@ func TestUpgradeNodeVersion(t *testing.T) { }}, currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - wantErr: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantErr: true, assertCorrectError: func(t *testing.T, err error) bool { return assert.ErrorIs(t, err, ErrInProgress) }, @@ -158,9 +239,13 @@ func TestUpgradeNodeVersion(t *testing.T) { }}, currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - force: true, - wantUpdate: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + force: true, + wantUpdate: true, }, "get error": { conf: func() *config.Config { @@ -173,7 +258,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, getCRErr: assert.AnError, @@ -194,7 +280,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -216,7 +303,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -234,7 +322,8 @@ func TestUpgradeNodeVersion(t *testing.T) { badImageVersion: "v3.2.1", kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -255,7 +344,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: false, @@ -276,7 +366,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: false, // because customClient is used @@ -346,6 +437,12 @@ func TestUpgradeNodeVersion(t *testing.T) { return } assert.NoError(err) + // The ConfigMap only exists in the updatedConfigMaps map it needed to remove the Konnectivity values + if strings.Contains(tc.kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") { + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config") + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file") + } }) } }