Merge pull request #16814 from justinsb/bring_up_etcd

metal: assume etcd discovery is external

pkg/model/components/etcdmanager/model.go

@@ -539,7 +539,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance
 			staticConfig.Nodes = append(staticConfig.Nodes, StaticConfigNode{
 				ID: fmt.Sprintf("%s--%s--%d", b.Cluster.Name, etcdCluster.Name, 0),
 				// TODO: Support multiple control-plane nodes (will be interesting!)
-				IP: []string{"127.0.0.1"},
+				IP: []string{"node0" + "." + etcdCluster.Name + "." + b.Cluster.Name},
 			})
 			b, err := json.Marshal(staticConfig)
 			if err != nil {

tests/e2e/scenarios/bare-metal/dump-artifacts

@@ -50,7 +50,10 @@ for vm in 0 1 2; do
   ip="10.123.45.1${vm}"
   vm_name="vm${vm}"
   mkdir -p ${ARTIFACTS}/vms/${vm_name}/logs/
-  scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected]:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true
-  ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kubelet.service || true
-  ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] journalctl --no-pager -u kops-configuration 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kops-configuration.service || true
+  mkdir -p ${ARTIFACTS}/vms/${vm_name}/manifests/
+  scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true
+  scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/var/log/kube* ${ARTIFACTS}/vms/${vm_name}/logs/ || true
+  scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/etc/kubernetes/manifests/* ${ARTIFACTS}/vms/${vm_name}/manifests/ || true
+  ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip} journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kubelet.service || true
+  ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip} journalctl --no-pager -u kops-configuration 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kops-configuration.service || true
 done

tests/e2e/scenarios/bare-metal/run-test

@@ -94,6 +94,10 @@ ${KOPS} edit cluster metal.k8s.local  --set spec.api.publicName=10.123.45.10
 # Use latest etcd-manager image (while we're adding features)
 ${KOPS} edit cluster metal.k8s.local --set 'spec.etcdClusters[*].manager.image=us-central1-docker.pkg.dev/k8s-staging-images/etcd-manager/etcd-manager-static:latest'
 
+# Use 1.31 kubernetes so we get kube-apiserver fixes
+export KOPS_RUN_TOO_NEW_VERSION=1
+"${KOPS}" edit cluster metal.k8s.local "--set=cluster.spec.kubernetesVersion=1.31.0"
+
 # List clusters
 ${KOPS} get cluster
 ${KOPS} get cluster -oyaml
@@ -113,7 +117,27 @@ ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519
 # Enroll the control-plane VM
 ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=8
 
-echo "Waiting 60 seconds for kube to start"
-sleep 60
+# Manual creation of "volumes" for etcd, and setting up peer nodes
+cat <<EOF | ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] tee -a /etc/hosts
+
+# Hosts added for etcd discovery
+10.123.45.10 node0.main.metal.k8s.local
+10.123.45.10 node0.events.metal.k8s.local
+EOF
+
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] cat /etc/hosts
+
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] mkdir -p /mnt/disks/metal.k8s.local--main--0/mnt
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] touch    /mnt/disks/metal.k8s.local--main--0/mnt/please-create-new-cluster
+
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] mkdir -p /mnt/disks/metal.k8s.local--events--0/mnt
+ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 [email protected] touch    /mnt/disks/metal.k8s.local--events--0/mnt/please-create-new-cluster
+
+
+echo "Waiting 300 seconds for kube to start"
+sleep 300
+
+kubectl get nodes
+kubectl get pods -A
 
 echo "Test successful"