Skip to content

Commit

Permalink
cli: reconcile kubeadm cm to not have konnectivity
Browse files Browse the repository at this point in the history
  • Loading branch information
@3u13r
3u13r committed Nov 24, 2023
1 parent 56fa7c5 commit 1fc77d8
Show file tree
Hide file tree
Showing 3 changed files with 159 additions and 16 deletions.
1 change: 1 addition & 0 deletions internal/kubecmd/BUILD.bazel
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ go_test(
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_sigs_yaml//:yaml",
],
)
45 changes: 45 additions & 0 deletions internal/kubecmd/kubecmd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"io"
"slices"
"sort"
"strings"
"time"
Expand Down Expand Up @@ -128,6 +129,12 @@ func (k *KubeCmd) UpgradeNodeVersion(ctx context.Context, conf *config.Config, f
case err != nil:
return fmt.Errorf("updating image version: %w", err)
}

// TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released.
if err := k.reconcileKubeadmConfigMap(ctx); err != nil {
return fmt.Errorf("reconciling kubeadm config: %w", err)
}

k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.Version())
nodeVersion.Spec.ImageReference = imageReference
nodeVersion.Spec.ImageVersion = imageVersion.Version()
Expand Down Expand Up @@ -393,6 +400,44 @@ func (k *KubeCmd) applyNodeVersion(ctx context.Context, nodeVersion updatev1alph
return updatedNodeVersion, err
}

func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error {
clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx)
if err != nil {
return fmt.Errorf("getting ClusterConfig: %w", err)
}

for i, v := range clusterConfiguration.APIServer.ExtraVolumes {
if v.Name == "konnectivity-uds" {
clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1)
}
}
for i, v := range clusterConfiguration.APIServer.ExtraVolumes {
if v.Name == "egress-config" {
clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1)
}
}
delete(clusterConfiguration.APIServer.ExtraArgs, "egress-selector-config-file")

newConfigYAML, err := yaml.Marshal(clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}

if kubeadmConfig.Data[constants.ClusterConfigurationKey] == string(newConfigYAML) {
k.log.Debugf("No changes to kubeadm config required")
return nil
}

kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debugf("Triggering kubeadm config update now")
if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
}

fmt.Fprintln(k.outWriter, "Successfully reconciled the cluster's kubeadm config")
return nil
}

// isValidImageUpdate checks if the new image version is a valid upgrade, and there is no upgrade already running.
func (k *KubeCmd) isValidImageUpgrade(nodeVersion updatev1alpha1.NodeVersion, newImageVersion string, force bool) error {
if !force {
Expand Down
129 changes: 113 additions & 16 deletions internal/kubecmd/kubecmd_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"encoding/json"
"errors"
"io"
"strings"
"testing"
"time"

Expand All @@ -34,9 +35,61 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)

func TestUpgradeNodeVersion(t *testing.T) {
clusterConf := kubeadmv1beta3.ClusterConfiguration{
APIServer: kubeadmv1beta3.APIServer{
ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{
ExtraArgs: map[string]string{},
ExtraVolumes: []kubeadmv1beta3.HostPathMount{},
},
},
}

clusterConfBytes, err := json.Marshal(clusterConf)
require.NoError(t, err)
validKubeadmConfig := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeadmConfigMap,
},
Data: map[string]string{
constants.ClusterConfigurationKey: string(clusterConfBytes),
},
}

clusterConfWithKonnectivity := kubeadmv1beta3.ClusterConfiguration{
APIServer: kubeadmv1beta3.APIServer{
ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{
ExtraArgs: map[string]string{
"egress-selector-config-file": "/etc/kubernetes/egress-selector-config-file.yaml",
},
ExtraVolumes: []kubeadmv1beta3.HostPathMount{
{
Name: "egress-config",
HostPath: "/etc/kubernetes/egress-selector-config-file.yaml",
},
{
Name: "konnectivity-uds",
HostPath: "/some/path/to/konnectivity-uds",
},
},
},
},
}

clusterConfBytesWithKonnectivity, err := json.Marshal(clusterConfWithKonnectivity)
require.NoError(t, err)
validKubeadmConfigWithKonnectivity := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeadmConfigMap,
},
Data: map[string]string{
constants.ClusterConfigurationKey: string(clusterConfBytesWithKonnectivity),
},
}

testCases := map[string]struct {
kubectl *stubKubectl
conditions []metav1.Condition
Expand All @@ -63,7 +116,25 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
},
"success with konnectivity migration": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfigWithKonnectivity,
},
},
wantUpdate: true,
Expand All @@ -79,7 +150,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -100,7 +172,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -119,8 +192,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
wantErr: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
Expand All @@ -139,8 +216,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
}},
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
wantErr: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
return assert.ErrorIs(t, err, ErrInProgress)
},
Expand All @@ -158,9 +239,13 @@ func TestUpgradeNodeVersion(t *testing.T) {
}},
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
force: true,
wantUpdate: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
force: true,
wantUpdate: true,
},
"get error": {
conf: func() *config.Config {
Expand All @@ -173,7 +258,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
getCRErr: assert.AnError,
Expand All @@ -194,7 +280,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -216,7 +303,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -234,7 +322,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
badImageVersion: "v3.2.1",
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -255,7 +344,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false,
Expand All @@ -276,7 +366,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false, // because customClient is used
Expand Down Expand Up @@ -346,6 +437,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
return
}
assert.NoError(err)
// The ConfigMap only exists in the updatedConfigMaps map it needed to remove the Konnectivity values
if strings.Contains(tc.kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") {
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file")
}
})
}
}
Expand Down

0 comments on commit 1fc77d8

Please sign in to comment.