diff --git a/internal/kubecmd/kubecmd.go b/internal/kubecmd/kubecmd.go index 6630d3544f..e422f3dc34 100644 --- a/internal/kubecmd/kubecmd.go +++ b/internal/kubecmd/kubecmd.go @@ -22,6 +22,7 @@ import ( "errors" "fmt" "io" + "slices" "sort" "strings" "time" @@ -128,6 +129,11 @@ func (k *KubeCmd) UpgradeNodeVersion(ctx context.Context, conf *config.Config, f case err != nil: return fmt.Errorf("updating image version: %w", err) } + + if err := k.reconcileKubeadmConfigMap(ctx); err != nil { + return fmt.Errorf("reconciling kubeadm config: %w", err) + } + k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.Version()) nodeVersion.Spec.ImageReference = imageReference nodeVersion.Spec.ImageVersion = imageVersion.Version() @@ -393,6 +399,39 @@ func (k *KubeCmd) applyNodeVersion(ctx context.Context, nodeVersion updatev1alph return updatedNodeVersion, err } +func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error { + clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx) + if err != nil { + return fmt.Errorf("getting ClusterConfig: %w", err) + } + + for i, v := range clusterConfiguration.APIServer.ExtraVolumes { + if v.Name == "konnectivity-uds" { + clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) + } + } + for i, v := range clusterConfiguration.APIServer.ExtraVolumes { + if v.Name == "egress-config" { + clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1) + } + } + delete(clusterConfiguration.APIServer.ExtraArgs, "egress-selector-config-file") + + newConfigYAML, err := yaml.Marshal(clusterConfiguration) + if err != nil { + return fmt.Errorf("marshaling ClusterConfiguration: %w", err) + } + + kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML) + k.log.Debugf("Triggering kubeadm config update now") + if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil { + return fmt.Errorf("setting new kubeadm config: %w", err) + } + + fmt.Fprintln(k.outWriter, "Successfully reconciled the cluster's kubeadm config") + return nil +} + // isValidImageUpdate checks if the new image version is a valid upgrade, and there is no upgrade already running. func (k *KubeCmd) isValidImageUpgrade(nodeVersion updatev1alpha1.NodeVersion, newImageVersion string, force bool) error { if !force { diff --git a/internal/kubecmd/kubecmd_test.go b/internal/kubecmd/kubecmd_test.go index c905e38cf4..84b064c061 100644 --- a/internal/kubecmd/kubecmd_test.go +++ b/internal/kubecmd/kubecmd_test.go @@ -34,9 +34,61 @@ import ( "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3" ) func TestUpgradeNodeVersion(t *testing.T) { + clusterConf := kubeadmv1beta3.ClusterConfiguration{ + APIServer: kubeadmv1beta3.APIServer{ + ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ + ExtraArgs: map[string]string{}, + ExtraVolumes: []kubeadmv1beta3.HostPathMount{}, + }, + }, + } + + clusterConfBytes, err := json.Marshal(clusterConf) + require.NoError(t, err) + validKubeadmConfig := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeadmConfigMap, + }, + Data: map[string]string{ + constants.ClusterConfigurationKey: string(clusterConfBytes), + }, + } + + clusterConfWithKonnectivity := kubeadmv1beta3.ClusterConfiguration{ + APIServer: kubeadmv1beta3.APIServer{ + ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{ + ExtraArgs: map[string]string{ + "egress-selector-config-file": "/etc/kubernetes/egress-selector-config-file.yaml", + }, + ExtraVolumes: []kubeadmv1beta3.HostPathMount{ + { + Name: "egress-config", + HostPath: "/etc/kubernetes/egress-selector-config-file.yaml", + }, + { + Name: "konnectivity-uds", + HostPath: "/some/path/to/konnectivity-uds", + }, + }, + }, + }, + } + + clusterConfBytesWithKonnectivity, err := json.Marshal(clusterConfWithKonnectivity) + require.NoError(t, err) + validKubeadmConfigWithKonnectivity := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: constants.KubeadmConfigMap, + }, + Data: map[string]string{ + constants.ClusterConfigurationKey: string(clusterConfBytesWithKonnectivity), + }, + } + testCases := map[string]struct { kubectl *stubKubectl conditions []metav1.Condition @@ -63,7 +115,25 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantUpdate: true, + }, + "success with konnectivity migration": { + conf: func() *config.Config { + conf := config.Default() + conf.Image = "v1.2.3" + conf.KubernetesVersion = supportedValidK8sVersions()[1] + return conf + }(), + currentImageVersion: "v1.2.2", + currentClusterVersion: supportedValidK8sVersions()[0], + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfigWithKonnectivity, }, }, wantUpdate: true, @@ -79,7 +149,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -100,7 +171,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -119,8 +191,12 @@ func TestUpgradeNodeVersion(t *testing.T) { }(), currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - wantErr: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantErr: true, assertCorrectError: func(t *testing.T, err error) bool { var upgradeErr *compatibility.InvalidUpgradeError return assert.ErrorAs(t, err, &upgradeErr) @@ -139,8 +215,12 @@ func TestUpgradeNodeVersion(t *testing.T) { }}, currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - wantErr: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + wantErr: true, assertCorrectError: func(t *testing.T, err error) bool { return assert.ErrorIs(t, err, ErrInProgress) }, @@ -158,9 +238,13 @@ func TestUpgradeNodeVersion(t *testing.T) { }}, currentImageVersion: "v1.2.2", currentClusterVersion: supportedValidK8sVersions()[0], - kubectl: &stubKubectl{}, - force: true, - wantUpdate: true, + kubectl: &stubKubectl{ + configMaps: map[string]*corev1.ConfigMap{ + constants.KubeadmConfigMap: validKubeadmConfig, + }, + }, + force: true, + wantUpdate: true, }, "get error": { conf: func() *config.Config { @@ -173,7 +257,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, getCRErr: assert.AnError, @@ -194,7 +279,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -216,7 +302,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -234,7 +321,8 @@ func TestUpgradeNodeVersion(t *testing.T) { badImageVersion: "v3.2.1", kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: true, @@ -255,7 +343,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: false, @@ -276,7 +365,8 @@ func TestUpgradeNodeVersion(t *testing.T) { currentClusterVersion: supportedValidK8sVersions()[0], kubectl: &stubKubectl{ configMaps: map[string]*corev1.ConfigMap{ - constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`), + constants.KubeadmConfigMap: validKubeadmConfig, }, }, wantUpdate: false, // because customClient is used @@ -346,6 +436,9 @@ func TestUpgradeNodeVersion(t *testing.T) { return } assert.NoError(err) + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config") + assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file") }) } }