From 28bd1155779c56a7e6e332bb1cdf66d00498e304 Mon Sep 17 00:00:00 2001 From: justinsb Date: Fri, 6 Sep 2024 11:27:13 -0400 Subject: [PATCH] metal: assume etcd discovery is external localhost won't work for multiple nodes and turns out not to even work for one node. Assume that there is a discovery mechanism that sets up /etc/hosts (it might be static configuration). Add to metal test to get to kube-apiserver responding. --- pkg/model/components/etcdmanager/model.go | 2 +- tests/e2e/scenarios/bare-metal/dump-artifacts | 9 ++++-- tests/e2e/scenarios/bare-metal/run-test | 28 +++++++++++++++++-- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pkg/model/components/etcdmanager/model.go b/pkg/model/components/etcdmanager/model.go index c6df86ffe911c..07f7d04ddc50b 100644 --- a/pkg/model/components/etcdmanager/model.go +++ b/pkg/model/components/etcdmanager/model.go @@ -539,7 +539,7 @@ func (b *EtcdManagerBuilder) buildPod(etcdCluster kops.EtcdClusterSpec, instance staticConfig.Nodes = append(staticConfig.Nodes, StaticConfigNode{ ID: fmt.Sprintf("%s--%s--%d", b.Cluster.Name, etcdCluster.Name, 0), // TODO: Support multiple control-plane nodes (will be interesting!) - IP: []string{"127.0.0.1"}, + IP: []string{"node0" + "." + etcdCluster.Name + "." + b.Cluster.Name}, }) b, err := json.Marshal(staticConfig) if err != nil { diff --git a/tests/e2e/scenarios/bare-metal/dump-artifacts b/tests/e2e/scenarios/bare-metal/dump-artifacts index c92689ae82d7f..5af3b9b6df0ed 100755 --- a/tests/e2e/scenarios/bare-metal/dump-artifacts +++ b/tests/e2e/scenarios/bare-metal/dump-artifacts @@ -50,7 +50,10 @@ for vm in 0 1 2; do ip="10.123.45.1${vm}" vm_name="vm${vm}" mkdir -p ${ARTIFACTS}/vms/${vm_name}/logs/ - scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true - ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kubelet.service || true - ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 journalctl --no-pager -u kops-configuration 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kops-configuration.service || true + mkdir -p ${ARTIFACTS}/vms/${vm_name}/manifests/ + scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/var/log/etcd* ${ARTIFACTS}/vms/${vm_name}/logs/ || true + scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/var/log/kube* ${ARTIFACTS}/vms/${vm_name}/logs/ || true + scp -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip}:/etc/kubernetes/manifests/* ${ARTIFACTS}/vms/${vm_name}/manifests/ || true + ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip} journalctl --no-pager -u kubelet 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kubelet.service || true + ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@${ip} journalctl --no-pager -u kops-configuration 2>&1 > ${ARTIFACTS}/vms/${vm_name}/logs/kops-configuration.service || true done diff --git a/tests/e2e/scenarios/bare-metal/run-test b/tests/e2e/scenarios/bare-metal/run-test index f2b354ade6fa5..9f8acf3905565 100755 --- a/tests/e2e/scenarios/bare-metal/run-test +++ b/tests/e2e/scenarios/bare-metal/run-test @@ -94,6 +94,10 @@ ${KOPS} edit cluster metal.k8s.local --set spec.api.publicName=10.123.45.10 # Use latest etcd-manager image (while we're adding features) ${KOPS} edit cluster metal.k8s.local --set 'spec.etcdClusters[*].manager.image=us-central1-docker.pkg.dev/k8s-staging-images/etcd-manager/etcd-manager-static:latest' +# Use 1.31 kubernetes so we get kube-apiserver fixes +export KOPS_RUN_TOO_NEW_VERSION=1 +"${KOPS}" edit cluster metal.k8s.local "--set=cluster.spec.kubernetesVersion=1.31.0" + # List clusters ${KOPS} get cluster ${KOPS} get cluster -oyaml @@ -113,7 +117,27 @@ ssh-add ${REPO_ROOT}/.build/.ssh/id_ed25519 # Enroll the control-plane VM ${KOPS} toolbox enroll --cluster metal.k8s.local --instance-group control-plane-main --host 10.123.45.10 --v=8 -echo "Waiting 60 seconds for kube to start" -sleep 60 +# Manual creation of "volumes" for etcd, and setting up peer nodes +cat <