Merge pull request #16805 from justinsb/etcd_manager_static_config

etcd manager static config

pkg/flagbuilder/build_flags.go

@@ -143,20 +143,19 @@ func BuildFlagsList(options interface{}) ([]string, error) {
 		case string:
 			vString := fmt.Sprintf("%v", v)
 			if vString != "" && vString != flagEmpty {
-				flag = fmt.Sprintf("--%s=%s", flagName, vString)
+				flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString))
 			}
 
 		case *string:
 			if v != nil {
 				// If flagIncludeEmpty is specified, include anything, including empty strings. Otherwise, behave
 				// just like the string case above.
+				vString := fmt.Sprintf("%v", *v)
 				if flagIncludeEmpty {
-					vString := fmt.Sprintf("%v", *v)
-					flag = fmt.Sprintf("--%s=%s", flagName, vString)
+					flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString))
 				} else {
-					vString := fmt.Sprintf("%v", *v)
 					if vString != "" && vString != flagEmpty {
-						flag = fmt.Sprintf("--%s=%s", flagName, vString)
+						flag = fmt.Sprintf("--%s=%s", flagName, maybeQuote(vString))
 					}
 				}
 			}
@@ -214,3 +213,10 @@ func BuildFlagsList(options interface{}) ([]string, error) {
 
 	return flags, nil
 }
+
+func maybeQuote(s string) string {
+	if strings.Contains(s, "\"") {
+		return fmt.Sprintf("%q", s)
+	}
+	return s
+}

pkg/model/components/etcdmanager/model.go

@@ -528,7 +528,24 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
 
 		case kops.CloudProviderMetal:
 			config.VolumeProvider = "external"
-			// TODO: Use static configuration here?
+			config.BackupStore = "file:///mnt/disks/backups"
+			config.VolumeTag = []string{
+				fmt.Sprintf("%s--%s--", b.Cluster.Name, etcdCluster.Name),
+			}
+
+			staticConfig := &StaticConfig{
+				EtcdVersion: etcdCluster.Version,
+			}
+			staticConfig.Nodes = append(staticConfig.Nodes, StaticConfigNode{
+				ID: fmt.Sprintf("%s--%s--%d", b.Cluster.Name, etcdCluster.Name, 0),
+				// TODO: Support multiple control-plane nodes (will be interesting!)
+				IP: []string{"127.0.0.1"},
+			})
+			b, err := json.Marshal(staticConfig)
+			if err != nil {
+				return nil, fmt.Errorf("building static config: %w", err)
+			}
+			config.StaticConfig = string(b)
 
 		default:
 			return nil, fmt.Errorf("CloudProvider %q not supported with etcd-manager", b.Cluster.GetCloudProvider())
@@ -653,6 +670,19 @@ type config struct {
 	VolumeNameTag         string   `flag:"volume-name-tag"`
 	DNSSuffix             string   `flag:"dns-suffix"`
 	NetworkCIDR           *string  `flag:"network-cidr"`
+
+	// StaticConfig enables running with a fixed etcd cluster configuration.
+	StaticConfig string `flag:"static-config"`
+}
+
+type StaticConfig struct {
+	EtcdVersion string             `json:"etcdVersion,omitempty"`
+	Nodes       []StaticConfigNode `json:"nodes,omitempty"`
+}
+
+type StaticConfigNode struct {
+	ID string   `json:"id,omitempty"`
+	IP []string `json:"ip,omitempty"`
 }
 
 // SelectorForCluster returns the selector that should be used to select our pods (from services)

tests/e2e/scenarios/bare-metal/dump-artifacts

@@ -51,4 +51,5 @@ for vm in 0 1 2; do
   vm_name="vm${vm}"
   mkdir -p ${ARTIFACTS}/vms/${vm_name}/logs/
   scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected]:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true
+  ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/journal-kubelet.service || true
 done

tests/e2e/scenarios/bare-metal/run-test

@@ -81,6 +81,9 @@ ${KOPS} create cluster --cloud=metal metal.k8s.local --zones main
 # TODO: is this the best option?
 ${KOPS} edit cluster metal.k8s.local  --set spec.api.publicName=10.123.45.10
 
+# Use latest etcd-manager image (while we're adding features)
+${KOPS} edit cluster metal.k8s.local --set 'spec.etcdClusters[*].manager.image=us-central1-docker.pkg.dev/k8s-staging-images/etcd-manager/etcd-manager-static:latest'
+
 # List clusters
 ${KOPS} get cluster
 ${KOPS} get cluster -oyaml
@@ -100,4 +103,7 @@ ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
 # Enroll the control-plane VM
 ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=8
 
+echo "Waiting 60 seconds for kube to start"
+sleep 60
+
 echo "Test successful"