From ef7a08c58b2f017105c99cc47c3d1dd0a2ad26f4 Mon Sep 17 00:00:00 2001 From: justinsb Date: Wed, 28 Aug 2024 06:53:58 -0400 Subject: [PATCH] tests: create basic test for creating VMs on github actions We try to bring up 3 VMs and SSH to them. --- .github/workflows/main.yml | 17 ++ tests/e2e/scenarios/bare-metal/cleanup | 42 +++++ tests/e2e/scenarios/bare-metal/run-test | 46 ++++++ tests/e2e/scenarios/bare-metal/start-vms | 195 +++++++++++++++++++++++ tools/metal/dhcp/go.mod | 12 ++ tools/metal/dhcp/go.sum | 28 ++++ tools/metal/dhcp/main.go | 123 ++++++++++++++ 7 files changed, 463 insertions(+) create mode 100755 tests/e2e/scenarios/bare-metal/cleanup create mode 100755 tests/e2e/scenarios/bare-metal/run-test create mode 100755 tests/e2e/scenarios/bare-metal/start-vms create mode 100644 tools/metal/dhcp/go.mod create mode 100644 tools/metal/dhcp/go.sum create mode 100644 tools/metal/dhcp/main.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8e86623fb015c..94a88c93b7f3a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,3 +80,20 @@ jobs: working-directory: ${{ env.GOPATH }}/src/k8s.io/kops run: | make quick-ci + + tests-e2e-scenarios-bare-metal: + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 + with: + path: ${{ env.GOPATH }}/src/k8s.io/kops + + - name: Set up go + uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 + with: + go-version-file: '${{ env.GOPATH }}/src/k8s.io/kops/go.mod' + + - name: tests/e2e/scenarios/bare-metal/run-test + working-directory: ${{ env.GOPATH }}/src/k8s.io/kops + run: | + tests/e2e/scenarios/bare-metal/run-test diff --git a/tests/e2e/scenarios/bare-metal/cleanup b/tests/e2e/scenarios/bare-metal/cleanup new file mode 100755 index 0000000000000..2a8e80b64897e --- /dev/null +++ b/tests/e2e/scenarios/bare-metal/cleanup @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +REPO_ROOT=$(git rev-parse --show-toplevel) +cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal + +cd ${REPO_ROOT} + +systemctl disable --user qemu-vm0 || true +systemctl disable --user qemu-vm1 || true +systemctl disable --user qemu-vm2 || true + +systemctl stop --user qemu-vm0 || true +systemctl stop --user qemu-vm1 || true +systemctl stop --user qemu-vm2 || true + +sudo ip link del dev tap-vm0 || true +sudo ip link del dev tap-vm1 || true +sudo ip link del dev tap-vm2 || true + +rm -rf .build/vm0 +rm -rf .build/vm1 +rm -rf .build/vm2 + diff --git a/tests/e2e/scenarios/bare-metal/run-test b/tests/e2e/scenarios/bare-metal/run-test new file mode 100755 index 0000000000000..3063495f4057d --- /dev/null +++ b/tests/e2e/scenarios/bare-metal/run-test @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +REPO_ROOT=$(git rev-parse --show-toplevel) +cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal + +function cleanup() { + echo "running cleanup" + ./cleanup || true +} + +trap cleanup EXIT + +./start-vms + +echo "Waiting 30 seconds for VMs to start" +#sleep 30 +sleep 15 + +# Remove from known-hosts in case of reuse +ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.10 +ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.11 +ssh-keygen -f ~/.ssh/known_hosts -R 10.123.45.12 + +ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.10 uptime +ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.11 uptime +ssh -o StrictHostKeyChecking=accept-new -i ${REPO_ROOT}/.build/.ssh/id_ed25519 root@10.123.45.12 uptime + diff --git a/tests/e2e/scenarios/bare-metal/start-vms b/tests/e2e/scenarios/bare-metal/start-vms new file mode 100755 index 0000000000000..bc12dd01d9b59 --- /dev/null +++ b/tests/e2e/scenarios/bare-metal/start-vms @@ -0,0 +1,195 @@ +#!/usr/bin/env bash + +# Copyright 2024 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail +set -o xtrace + +REPO_ROOT=$(git rev-parse --show-toplevel) +cd ${REPO_ROOT}/tests/e2e/scenarios/bare-metal + +WORKDIR=${REPO_ROOT}/.build + +# Create SSH keys +mkdir -p ${WORKDIR}/.ssh +if [[ ! -f ${WORKDIR}/.ssh/id_ed25519 ]]; then + ssh-keygen -t ed25519 -f ${WORKDIR}/.ssh/id_ed25519 -N "" +fi + +# Build software we need +cd ${REPO_ROOT}/tools/metal/dhcp +go build -o ${WORKDIR}/dhcp . +# Give permission to listen on ports < 1024 (some of like a partial suid binary) +sudo setcap cap_net_bind_service=ep ${WORKDIR}/dhcp + +# Install software we need +if ! genisoimage --version; then + echo "Installing genisoimage" + sudo apt-get install --yes genisoimage +fi +if ! qemu-img --version; then + echo "Installing qemu-img (via qemu-utils)" + sudo apt-get install --yes qemu-utils +fi + +# Download boot disk +cd ${WORKDIR} +if [[ ! -f debian-12-generic-amd64-20240702-1796.qcow2 ]]; then + echo "Downloading debian-12-generic-amd64-20240702-1796.qcow2" + wget --no-verbose -N https://cloud.debian.org/images/cloud/bookworm/20240702-1796/debian-12-generic-amd64-20240702-1796.qcow2 +fi + +# Create bridge +bridge_name=br0 +if (! ip link show ${bridge_name}); then + sudo ip link add ${bridge_name} type bridge + sudo ip address add 10.123.45.0/24 dev ${bridge_name} + + sudo ip link add name loop-${bridge_name} type dummy + sudo ip addr add 10.123.45.1/24 dev loop-${bridge_name} + + sudo ip link set dev loop-${bridge_name} master ${bridge_name} + + # Enable packets from one VM on the bridge to another + sudo iptables -A FORWARD -i ${bridge_name} -o ${bridge_name} -j ACCEPT + + # Enable packets from a VM to reach the real internet via NAT + sudo iptables -t nat -A POSTROUTING -s 10.123.45.0/24 ! -o ${bridge_name} -j MASQUERADE + sudo iptables -A FORWARD -o ${bridge_name} -m state --state RELATED,ESTABLISHED -j ACCEPT + sudo iptables -A FORWARD -i ${bridge_name} ! -o ${bridge_name} -j ACCEPT + + sudo ip link set dev ${bridge_name} up +fi + +# TODO: Create tuntap with user $(whoami)? + + +function start_dhcp() { +# Great guide here: https://wiki.gentoo.org/wiki/QEMU/Options + mkdir -p ~/.config/systemd/user + cat < ~/.config/systemd/user/qemu-dhcp.service +[Unit] +Description=qemu-dhcp +After=network.target + +[Service] +EnvironmentFile=/etc/environment +Type=exec +WorkingDirectory=${WORKDIR}/ +ExecStart=${WORKDIR}/dhcp +Restart=always + +[Install] +WantedBy=default.target +EOF + + systemctl --user daemon-reload + systemctl --user enable --now qemu-dhcp.service +} + +function run_vm() { + vm_name=$1 + mac=$2 + #ssh_port=$2 + + mkdir ${WORKDIR}/${vm_name}/ + cd ${WORKDIR}/${vm_name} + PUBKEY=$(cat ${WORKDIR}/.ssh/id_ed25519.pub) + + cat < user-data +#cloud-config +users: + - name: my_user + groups: adm, cdrom, sudo, dip, plugdev, lxd + shell: /bin/bash + sudo: ALL=(ALL) NOPASSWD:ALL + ssh_authorized_keys: + - ${PUBKEY} + - name: root + ssh_authorized_keys: + - ${PUBKEY} + +hostname: ${vm_name} + +manage_etc_hosts: true + +locale: en_US.UTF-8 +timezone: Europe/Berlin +EOF + + cat < meta-data +instance-id: ${vm_name} +local-hostname: cloudimg +EOF + + genisoimage -output seed.iso -volid cidata -joliet -rock user-data meta-data + + qemu-img create -b ../debian-12-generic-amd64-20240702-1796.qcow2 -F qcow2 -f qcow2 root.qcow2 20G + + sudo ip tuntap add dev tap-${vm_name} mode tap + sudo ip link set dev tap-${vm_name} master ${bridge_name} + sudo ip link set dev tap-${vm_name} up + +# Great guide here: https://wiki.gentoo.org/wiki/QEMU/Options + mkdir -p ~/.config/systemd/user + cat < ~/.config/systemd/user/qemu-${vm_name}.service +[Unit] +Description=qemu-${vm_name} +After=network.target + +[Service] +EnvironmentFile=/etc/environment +Type=exec +WorkingDirectory=${WORKDIR}/${vm_name}/ +ExecStart=qemu-system-x86_64 \ + -machine type=q35,accel=kvm \ + -object rng-random,id=rng0,filename=/dev/urandom -device virtio-rng-pci,rng=rng0 \ + -m 1024 \ + -cpu host \ + -smp 4 \ + -nographic \ + -netdev tap,ifname=tap-${vm_name},id=net0,script=no,downscript=no \ + -device virtio-net-pci,netdev=net0,mac=${mac} \ + -device virtio-scsi-pci,id=scsi0 \ + -drive file=root.qcow2,id=hdd0,if=none \ + -device scsi-hd,drive=hdd0,bus=scsi0.0 \ + -drive id=cdrom0,if=none,format=raw,readonly=on,file=seed.iso \ + -device scsi-cd,bus=scsi0.0,drive=cdrom0 +Restart=always + +[Install] +WantedBy=default.target +EOF + +# TODO: +# -serial mon:stdio -append 'console=ttyS0' + +# TODO: control mac ,mac=${mac} + + systemctl --user daemon-reload + systemctl --user enable --now qemu-${vm_name}.service +} + + +start_dhcp + +# Note: not all mac addresses are valid; 52:54:00 is the prefix reserved for qemu +run_vm vm0 52:54:00:44:55:0a +run_vm vm1 52:54:00:44:55:0b +run_vm vm2 52:54:00:44:55:0c + +exit 0 diff --git a/tools/metal/dhcp/go.mod b/tools/metal/dhcp/go.mod new file mode 100644 index 0000000000000..92c4ea978588e --- /dev/null +++ b/tools/metal/dhcp/go.mod @@ -0,0 +1,12 @@ +module github.com/kubernetes/kops/tools/metal/dhcp + +go 1.22.6 + +require github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5 + +require ( + github.com/josharian/native v1.1.0 // indirect + github.com/pierrec/lz4/v4 v4.1.14 // indirect + github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 // indirect + golang.org/x/sys v0.18.0 // indirect +) diff --git a/tools/metal/dhcp/go.sum b/tools/metal/dhcp/go.sum new file mode 100644 index 0000000000000..a796094a6f258 --- /dev/null +++ b/tools/metal/dhcp/go.sum @@ -0,0 +1,28 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5 h1:GkMacU5ftc+IEg1449N3UEy2XLDz58W4fkrRu2fibb8= +github.com/insomniacslk/dhcp v0.0.0-20240812123929-b105c29bd1b5/go.mod h1:KclMyHxX06VrVr0DJmeFSUb1ankt7xTfoOA35pCkoic= +github.com/josharian/native v1.0.1-0.20221213033349-c1e37c09b531/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY= +github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4= +github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= +github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA= +github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= +github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923 h1:tHNk7XK9GkmKUR6Gh8gVBKXc2MVSZ4G/NnWLtzw4gNA= +github.com/u-root/uio v0.0.0-20230220225925-ffce2a382923/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/metal/dhcp/main.go b/tools/metal/dhcp/main.go new file mode 100644 index 0000000000000..da6e558db75ad --- /dev/null +++ b/tools/metal/dhcp/main.go @@ -0,0 +1,123 @@ +/* +Copyright 2024 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "log" + "net" + "os" + + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/server4" +) + +func main() { + err := run(context.Background()) + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +func handler(conn net.PacketConn, peer net.Addr, m *dhcpv4.DHCPv4) { + // this function will just print the received DHCPv4 message, without replying + if m == nil { + log.Printf("Packet is nil!") + return + } + + fmt.Printf("got dhcp packet: messageType=%v, mac=%v\n", m.MessageType(), m.ClientHWAddr) + + if m.OpCode != dhcpv4.OpcodeBootRequest { + log.Printf("Not a BootRequest!") + return + } + modifiers := []dhcpv4.Modifier{} + + switch mt := m.MessageType(); mt { + case dhcpv4.MessageTypeDiscover: + modifiers = append(modifiers, dhcpv4.WithMessageType(dhcpv4.MessageTypeOffer)) + // reply.UpdateOption(dhcpv4.OptServerIdentifier(net.IP{10, 20, 30, 1})) + // reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer)) + case dhcpv4.MessageTypeRequest: + modifiers = append(modifiers, dhcpv4.WithMessageType(dhcpv4.MessageTypeAck)) + // reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck)) + default: + log.Printf("Unhandled message type: %v", mt) + return + } + + if len(m.ClientHWAddr) != 6 { + log.Printf("unexpected mac address %v (expected length 6)", m.ClientHWAddr) + return + } + + clientIP := net.IP{10, 123, 45, m.ClientHWAddr[5]} + serverIP := net.IP{10, 123, 45, 1} + + router := serverIP + + modifiers = append(modifiers, dhcpv4.WithYourIP(clientIP)) + modifiers = append(modifiers, dhcpv4.WithDNS(net.IP{8, 8, 8, 8})) + modifiers = append(modifiers, dhcpv4.WithNetmask(net.IPMask{255, 255, 255, 0})) + modifiers = append(modifiers, dhcpv4.WithRouter(router)) + // modifiers = append(modifiers, dhcpv4.WithGatewayIP(net.IP{10, 20, 30, 1})) + modifiers = append(modifiers, dhcpv4.WithLeaseTime(60*60*4)) + modifiers = append(modifiers, dhcpv4.WithServerIP(serverIP)) + modifiers = append(modifiers, dhcpv4.WithOption(dhcpv4.OptServerIdentifier(serverIP))) + + reply, err := dhcpv4.NewReplyFromRequest(m, modifiers...) + if err != nil { + log.Printf("NewReplyFromRequest failed: %v", err) + return + } + // reply.UpdateOption(dhcpv4.OptServerIdentifier(net.IP{10, 20, 30, 1})) + // reply.UpdateOption(dhcpv4.OptClientIdentifier(net.IP{10, 20, 30, 4})) + + if _, err := conn.WriteTo(reply.ToBytes(), peer); err != nil { + log.Printf("Cannot reply to client: %v", err) + } +} + +func run(ctx context.Context) error { + // listenIP := "10.123.45.1" + listenIP := "0.0.0.0" + // listenInterface := "loop-br0" + listenInterface := "br0" + + ip := net.ParseIP(listenIP) + if ip == nil { + return fmt.Errorf("unable to parse IP %q", listenIP) + } + laddr := net.UDPAddr{ + IP: ip, + Port: 67, + } + fmt.Printf("listening on %v\n", listenInterface) + server, err := server4.NewServer(listenInterface, &laddr, handler) + if err != nil { + log.Fatal(err) + } + + // This never returns. If you want to do other stuff, dump it into a + // goroutine. + server.Serve() + + return nil +}