From f0555ca3bfb90f08c2d6f3e858f3e2c08148946b Mon Sep 17 00:00:00 2001 From: justinsb Date: Tue, 3 Sep 2024 11:55:16 -0400 Subject: [PATCH 1/2] tests: use latest dev etcd-manager image in bare-metal test At least while we are building out the static functionality. --- tests/e2e/scenarios/bare-metal/dump-artifacts | 1 + tests/e2e/scenarios/bare-metal/run-test | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/tests/e2e/scenarios/bare-metal/dump-artifacts b/tests/e2e/scenarios/bare-metal/dump-artifacts index a91824d9e7484..cec8a1db616d7 100755 --- a/tests/e2e/scenarios/bare-metal/dump-artifacts +++ b/tests/e2e/scenarios/bare-metal/dump-artifacts @@ -51,4 +51,5 @@ for vm in 0 1 2; do vm_name="vm${vm}" mkdir -p ${ARTIFACTS}/vms/${vm_name}/logs/ scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true + ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/journal-kubelet.service || true done diff --git a/tests/e2e/scenarios/bare-metal/run-test b/tests/e2e/scenarios/bare-metal/run-test index c5eeb6a72ae12..df12572593706 100755 --- a/tests/e2e/scenarios/bare-metal/run-test +++ b/tests/e2e/scenarios/bare-metal/run-test @@ -81,6 +81,9 @@ ${KOPS} create cluster --cloud=metal metal.k8s.local --zones main # TODO: is this the best option? ${KOPS} edit cluster metal.k8s.local --set spec.api.publicName=10.123.45.10 +# Use latest etcd-manager image (while we're adding features) +${KOPS} edit cluster metal.k8s.local --set 'spec.etcdClusters[*].manager.image=us-central1-docker.pkg.dev/k8s-staging-images/etcd-manager/etcd-manager-static:latest' + # List clusters ${KOPS} get cluster ${KOPS} get cluster -oyaml @@ -100,4 +103,7 @@ ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519 # Enroll the control-plane VM ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=8 +echo "Waiting 60 seconds for kube to start" +sleep 60 + echo "Test successful" \ No newline at end of file From ab0f6847d4ec5ef6e45548bf2a75d7158705f26a Mon Sep 17 00:00:00 2001 From: justinsb Date: Tue, 3 Sep 2024 12:40:35 -0400 Subject: [PATCH 2/2] bare-metal: configure etcd with static configuration Likely restricted to a single node for now (because of the need for well-known IP addresses) --- pkg/flagbuilder/build_flags.go | 16 ++++++++---- pkg/model/components/etcdmanager/model.go | 32 ++++++++++++++++++++++- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/pkg/flagbuilder/build_flags.go b/pkg/flagbuilder/build_flags.go index 5ec81806f0876..419b4eb76fa1a 100644 --- a/pkg/flagbuilder/build_flags.go +++ b/pkg/flagbuilder/build_flags.go @@ -143,20 +143,19 @@ func BuildFlagsList(options interface{}) ([]string, error) { case string: vString := fmt.Sprintf("%v", v) if vString != "" && vString != flagEmpty { - flag = fmt.Sprintf("--%s=%s", flagName, vString) + flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString)) } case *string: if v != nil { // If flagIncludeEmpty is specified, include anything, including empty strings. Otherwise, behave // just like the string case above. + vString := fmt.Sprintf("%v", *v) if flagIncludeEmpty { - vString := fmt.Sprintf("%v", *v) - flag = fmt.Sprintf("--%s=%s", flagName, vString) + flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString)) } else { - vString := fmt.Sprintf("%v", *v) if vString != "" && vString != flagEmpty { - flag = fmt.Sprintf("--%s=%s", flagName, vString) + flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString)) } } } @@ -214,3 +213,10 @@ func BuildFlagsList(options interface{}) ([]string, error) { return flags, nil } + +func maybeQuote(s string) string { + if strings.Contains(s, "\"") { + return fmt.Sprintf("%q", s) + } + return s +} diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index 68d8961d5dafc..c6df86ffe911c 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -528,7 +528,24 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance case kops.CloudProviderMetal: config.VolumeProvider = "external" - // TODO: Use static configuration here? + config.BackupStore = "file:///mnt/disks/backups" + config.VolumeTag = []string{ + fmt.Sprintf("%s--%s--", b.Cluster.Name, etcdCluster.Name), + } + + staticConfig := &StaticConfig{ + EtcdVersion: etcdCluster.Version, + } + staticConfig.Nodes = append(staticConfig.Nodes, StaticConfigNode{ + ID: fmt.Sprintf("%s--%s--%d", b.Cluster.Name, etcdCluster.Name, 0), + // TODO: Support multiple control-plane nodes (will be interesting!) + IP: []string{"127.0.0.1"}, + }) + b, err := json.Marshal(staticConfig) + if err != nil { + return nil, fmt.Errorf("building static config: %w", err) + } + config.StaticConfig = string(b) default: return nil, fmt.Errorf("CloudProvider %q not supported with etcd-manager", b.Cluster.GetCloudProvider()) @@ -653,6 +670,19 @@ type config struct { VolumeNameTag string `flag:"volume-name-tag"` DNSSuffix string `flag:"dns-suffix"` NetworkCIDR *string `flag:"network-cidr"` + + // StaticConfig enables running with a fixed etcd cluster configuration. + StaticConfig string `flag:"static-config"` +} + +type StaticConfig struct { + EtcdVersion string `json:"etcdVersion,omitempty"` + Nodes []StaticConfigNode `json:"nodes,omitempty"` +} + +type StaticConfigNode struct { + ID string `json:"id,omitempty"` + IP []string `json:"ip,omitempty"` } // SelectorForCluster returns the selector that should be used to select our pods (from services)