Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix Konnectivity migration #2633

Merged
merged 2 commits into from
Nov 24, 2023
Merged

Fix Konnectivity migration #2633

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
56fa7c5
helm: let cilium upgrade jump minor versions
3u13r Nov 22, 2023
1fc77d8
cli: reconcile kubeadm cm to not have konnectivity
3u13r Nov 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion internal/helm/actionfactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,11 @@ func (a actionFactory) appendNewAction(release Release, configTargetVersion semv
} else {
// This may break for external chart dependencies if we decide to upgrade more than one minor version at a time.
if err := newVersion.IsUpgradeTo(currentVersion); err != nil {
return fmt.Errorf("invalid upgrade for %s: %w", release.ReleaseName, err)
// TODO(3u13r): Remove when Constellation v2.14 is released.
// We need to ignore that we jump from Cilium v1.12 to v1.15-pre. We have verified that this works.
if !(errors.Is(err, compatibility.ErrMinorDrift) && release.ReleaseName == "cilium") {
return fmt.Errorf("invalid upgrade for %s: %w", release.ReleaseName, err)
}
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion internal/helm/helm_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ func TestHelmApply(t *testing.T) {
if tc.clusterCertManagerVersion != nil {
certManagerVersion = *tc.clusterCertManagerVersion
}
helmListVersion(lister, "cilium", "v1.12.1")
helmListVersion(lister, "cilium", "v1.15.0-pre.2")
helmListVersion(lister, "cert-manager", certManagerVersion)
helmListVersion(lister, "constellation-services", tc.clusterMicroServiceVersion)
helmListVersion(lister, "constellation-operators", tc.clusterMicroServiceVersion)
Expand Down
1 change: 1 addition & 0 deletions internal/kubecmd/BUILD.bazel
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ go_test(
"@io_k8s_apimachinery//pkg/apis/meta/v1/unstructured",
"@io_k8s_apimachinery//pkg/runtime",
"@io_k8s_apimachinery//pkg/runtime/schema",
"@io_k8s_kubernetes//cmd/kubeadm/app/apis/kubeadm/v1beta3",
"@io_k8s_sigs_yaml//:yaml",
],
)
45 changes: 45 additions & 0 deletions internal/kubecmd/kubecmd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"errors"
"fmt"
"io"
"slices"
"sort"
"strings"
"time"
Expand Down Expand Up @@ -128,6 +129,12 @@ func (k *KubeCmd) UpgradeNodeVersion(ctx context.Context, conf *config.Config, f
case err != nil:
return fmt.Errorf("updating image version: %w", err)
}

// TODO(3u13r): remove `reconcileKubeadmConfigMap` after v2.14.0 has been released.
if err := k.reconcileKubeadmConfigMap(ctx); err != nil {
return fmt.Errorf("reconciling kubeadm config: %w", err)
}

k.log.Debugf("Updating local copy of nodeVersion image version from %s to %s", nodeVersion.Spec.ImageVersion, imageVersion.Version())
nodeVersion.Spec.ImageReference = imageReference
nodeVersion.Spec.ImageVersion = imageVersion.Version()
Expand Down Expand Up @@ -393,6 +400,44 @@ func (k *KubeCmd) applyNodeVersion(ctx context.Context, nodeVersion updatev1alph
return updatedNodeVersion, err
}

func (k *KubeCmd) reconcileKubeadmConfigMap(ctx context.Context) error {
daniel-weisse marked this conversation as resolved.
Show resolved Hide resolved
clusterConfiguration, kubeadmConfig, err := k.getClusterConfiguration(ctx)
if err != nil {
return fmt.Errorf("getting ClusterConfig: %w", err)
}

for i, v := range clusterConfiguration.APIServer.ExtraVolumes {
if v.Name == "konnectivity-uds" {
clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1)
}
}
for i, v := range clusterConfiguration.APIServer.ExtraVolumes {
if v.Name == "egress-config" {
clusterConfiguration.APIServer.ExtraVolumes = slices.Delete(clusterConfiguration.APIServer.ExtraVolumes, i, i+1)
}
}
delete(clusterConfiguration.APIServer.ExtraArgs, "egress-selector-config-file")

newConfigYAML, err := yaml.Marshal(clusterConfiguration)
if err != nil {
return fmt.Errorf("marshaling ClusterConfiguration: %w", err)
}

if kubeadmConfig.Data[constants.ClusterConfigurationKey] == string(newConfigYAML) {
k.log.Debugf("No changes to kubeadm config required")
return nil
}

kubeadmConfig.Data[constants.ClusterConfigurationKey] = string(newConfigYAML)
k.log.Debugf("Triggering kubeadm config update now")
if _, err = k.kubectl.UpdateConfigMap(ctx, kubeadmConfig); err != nil {
return fmt.Errorf("setting new kubeadm config: %w", err)
}

fmt.Fprintln(k.outWriter, "Successfully reconciled the cluster's kubeadm config")
return nil
}

// isValidImageUpdate checks if the new image version is a valid upgrade, and there is no upgrade already running.
func (k *KubeCmd) isValidImageUpgrade(nodeVersion updatev1alpha1.NodeVersion, newImageVersion string, force bool) error {
if !force {
Expand Down
129 changes: 113 additions & 16 deletions internal/kubecmd/kubecmd_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"encoding/json"
"errors"
"io"
"strings"
"testing"
"time"

Expand All @@ -34,9 +35,61 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
kubeadmv1beta3 "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)

func TestUpgradeNodeVersion(t *testing.T) {
clusterConf := kubeadmv1beta3.ClusterConfiguration{
APIServer: kubeadmv1beta3.APIServer{
ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{
ExtraArgs: map[string]string{},
ExtraVolumes: []kubeadmv1beta3.HostPathMount{},
},
},
}

clusterConfBytes, err := json.Marshal(clusterConf)
require.NoError(t, err)
validKubeadmConfig := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeadmConfigMap,
},
Data: map[string]string{
constants.ClusterConfigurationKey: string(clusterConfBytes),
},
}

clusterConfWithKonnectivity := kubeadmv1beta3.ClusterConfiguration{
APIServer: kubeadmv1beta3.APIServer{
ControlPlaneComponent: kubeadmv1beta3.ControlPlaneComponent{
ExtraArgs: map[string]string{
"egress-selector-config-file": "/etc/kubernetes/egress-selector-config-file.yaml",
},
ExtraVolumes: []kubeadmv1beta3.HostPathMount{
{
Name: "egress-config",
HostPath: "/etc/kubernetes/egress-selector-config-file.yaml",
},
{
Name: "konnectivity-uds",
HostPath: "/some/path/to/konnectivity-uds",
},
},
},
},
}

clusterConfBytesWithKonnectivity, err := json.Marshal(clusterConfWithKonnectivity)
require.NoError(t, err)
validKubeadmConfigWithKonnectivity := &corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{
Name: constants.KubeadmConfigMap,
},
Data: map[string]string{
constants.ClusterConfigurationKey: string(clusterConfBytesWithKonnectivity),
},
}

testCases := map[string]struct {
kubectl *stubKubectl
conditions []metav1.Condition
Expand All @@ -63,7 +116,25 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
},
"success with konnectivity migration": {
conf: func() *config.Config {
conf := config.Default()
conf.Image = "v1.2.3"
conf.KubernetesVersion = supportedValidK8sVersions()[1]
return conf
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfigWithKonnectivity,
},
},
wantUpdate: true,
Expand All @@ -79,7 +150,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -100,7 +172,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -119,8 +192,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
}(),
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
wantErr: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
var upgradeErr *compatibility.InvalidUpgradeError
return assert.ErrorAs(t, err, &upgradeErr)
Expand All @@ -139,8 +216,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
}},
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
wantErr: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantErr: true,
assertCorrectError: func(t *testing.T, err error) bool {
return assert.ErrorIs(t, err, ErrInProgress)
},
Expand All @@ -158,9 +239,13 @@ func TestUpgradeNodeVersion(t *testing.T) {
}},
currentImageVersion: "v1.2.2",
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{},
force: true,
wantUpdate: true,
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
force: true,
wantUpdate: true,
},
"get error": {
conf: func() *config.Config {
Expand All @@ -173,7 +258,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
getCRErr: assert.AnError,
Expand All @@ -194,7 +280,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":true}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -216,7 +303,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -234,7 +322,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
badImageVersion: "v3.2.1",
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: true,
Expand All @@ -255,7 +344,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false,
Expand All @@ -276,7 +366,8 @@ func TestUpgradeNodeVersion(t *testing.T) {
currentClusterVersion: supportedValidK8sVersions()[0],
kubectl: &stubKubectl{
configMaps: map[string]*corev1.ConfigMap{
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.JoinConfigMap: newJoinConfigMap(`{"0":{"expected":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA","warnOnly":false}}`),
constants.KubeadmConfigMap: validKubeadmConfig,
},
},
wantUpdate: false, // because customClient is used
Expand Down Expand Up @@ -346,6 +437,12 @@ func TestUpgradeNodeVersion(t *testing.T) {
return
}
assert.NoError(err)
// The ConfigMap only exists in the updatedConfigMaps map it needed to remove the Konnectivity values
if strings.Contains(tc.kubectl.configMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds") {
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "konnectivity-uds")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-config")
assert.NotContains(tc.kubectl.updatedConfigMaps[constants.KubeadmConfigMap].Data[constants.ClusterConfigurationKey], "egress-selector-config-file")
}
})
}
}
Expand Down