Skip to content

neonvm: introduce CPU sysfs state scaling flow based on the vmSpec.cpuScalingMode #3804

neonvm: introduce CPU sysfs state scaling flow based on the vmSpec.cpuScalingMode

neonvm: introduce CPU sysfs state scaling flow based on the vmSpec.cpuScalingMode #3804

Workflow file for this run

name: e2e-test
on:
pull_request:
push:
branches:
- main
workflow_dispatch:
inputs:
kernel-image:
type: string
description: 'The kernel image to use for the VMs. If not specified, a kernel will be built from source'
required: false
cluster:
type: choice
description: 'The cluster to run the tests on'
options:
- k3d
- kind
default: k3d
workflow_call:
inputs:
tag:
type: string
description: 'Tag to use for images, skipping building'
required: false
push-yamls:
type: boolean
description: 'If true, pushes a tarball containing the rendered yaml manifests as an artifact'
required: false
env:
IMG_E2E_TEST: vm-postgres:15-bullseye
defaults:
run:
shell: bash -euo pipefail {0}
jobs:
get-tag:
outputs:
tag: ${{ inputs.tag || steps.get-tag.outputs.tag }}
runs-on: ubuntu-latest
steps:
- name: get tag
if: ${{ inputs.tag == '' }}
id: get-tag
env:
SHA: ${{ github.event.pull_request.head.sha || github.sha }}
run: |
test -n "$SHA"
sha="${SHA::7}"
echo "tag=$sha.$GITHUB_RUN_ID" | tee -a $GITHUB_OUTPUT
build-images:
needs: get-tag
uses: ./.github/workflows/build-images.yaml
with:
skip: ${{ inputs.tag != '' }}
tag: ${{ inputs.tag || needs.get-tag.outputs.tag }}
kernel-image: ${{ inputs.kernel-image }}
# note: setting to preserve runner pods will mean that if !skip, they'll be built with those
# settings and used properly in the tests. But if skip (because inputs.tag != ''), then this
# setting will have no effect and the release images will be normal.
controller-preserve-runner-pods: true
secrets: inherit
build-test-vm:
needs: get-tag
uses: ./.github/workflows/build-test-vm.yaml
with:
skip: ${{ inputs.tag != '' }}
tag: ${{ inputs.tag || needs.get-tag.outputs.tag }}
secrets: inherit
e2e-tests:
needs: [ build-images, build-test-vm ]
strategy:
fail-fast: false
matrix:
cluster:
- ${{ inputs.cluster || 'k3d' }}
runs-on: [ self-hosted, gen3, large ]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0 # fetch all, so that we also include tags
- uses: actions/setup-go@v5
with:
go-version-file: 'go.mod'
# Disable cache on self-hosted runners to avoid /usr/bin/tar errors, see https://github.com/actions/setup-go/issues/403
cache: false
# Sometimes setup-go gets stuck. Without this, it'll keep going until the job gets killed
timeout-minutes: 10
- name: Install dependencies
run: |
sudo apt install -y python3-venv
make e2e-tools
echo $(pwd)/bin >> $GITHUB_PATH
- name: Check dependencies
run: |
kubectl version --client --output=yaml
k3d version
kind version
kuttl version
docker version
- run: make render-release
env:
IMG_CONTROLLER: ${{ needs.build-images.outputs.controller }}
IMG_VXLAN_CONTROLLER: ${{ needs.build-images.outputs.vxlan-controller }}
IMG_RUNNER: ${{ needs.build-images.outputs.runner }}
IMG_SCHEDULER: ${{ needs.build-images.outputs.scheduler }}
IMG_AUTOSCALER_AGENT: ${{ needs.build-images.outputs.autoscaler-agent }}
- name: upload manifests
# nb: use format(..) to catch both inputs.push-yamls = true AND inputs.push-yamls = 'true'.
if: ${{ format('{0}', inputs.push-yamls) == 'true' }}
uses: actions/upload-artifact@v4
with:
name: rendered_manifests
# nb: prefix before wildcard is removed from the uploaded files, so the artifact should
# contain e.g.
# - autoscale-scheduler.yaml
# - autoscaler-agent.yaml
# ...
# ref https://github.com/actions/upload-artifact#upload-using-multiple-paths-and-exclusions
path: rendered_manifests/*
if-no-files-found: error
retention-days: 2 # minimum is 1 day; 0 is default. These are only used temporarily.
- name: set custom docker config directory
uses: neondatabase/dev-actions/set-docker-config-dir@6094485bf440001c94a94a3f9e221e81ff6b6193
- uses: docker/login-action@v3
with:
username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
# https://docs.k3s.io/installation/private-registry#registries-configuration-file
# https://github.com/neondatabase/autoscaling/issues/975
- name: set k3d registries.yaml
# TODO: Implement an equivalent for kind?
# Relevant docs seem to be here: https://kind.sigs.k8s.io/docs/user/private-registries
if: ${{ matrix.cluster == 'k3d' }}
env:
DOCKERHUB_USERNAME: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
DOCKERHUB_PASSWORD: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
run: |
{
echo "configs:"
echo " registry-1.docker.io:"
echo " auth:"
echo " username: $DOCKERHUB_USERNAME"
echo " password: $DOCKERHUB_PASSWORD"
} >> $(pwd)/k3d/registries.yaml
- run: make ${{ matrix.cluster }}-setup
env:
USE_REGISTRIES_FILE: true
- name: deploy components
timeout-minutes: 3
run: |
rendered () { echo "rendered_manifests/$1"; }
kubectl apply -f $(rendered multus.yaml)
kubectl -n kube-system rollout status daemonset kube-multus-ds
kubectl apply -f $(rendered whereabouts.yaml)
kubectl -n kube-system rollout status daemonset whereabouts
kubectl apply -f $(rendered neonvm-runner-image-loader.yaml)
kubectl -n neonvm-system rollout status daemonset neonvm-runner-image-loader
kubectl apply -f $(rendered neonvm.yaml)
kubectl -n neonvm-system rollout status daemonset neonvm-device-plugin
kubectl apply -f $(rendered neonvm-controller.yaml)
kubectl -n neonvm-system rollout status deployment neonvm-controller
kubectl apply -f $(rendered neonvm-vxlan-controller.yaml)
kubectl -n neonvm-system rollout status daemonset neonvm-vxlan-controller
kubectl apply -f $(rendered autoscale-scheduler.yaml)
kubectl -n kube-system rollout status deployment autoscale-scheduler
kubectl apply -f $(rendered autoscaler-agent.yaml)
kubectl -n kube-system rollout status daemonset autoscaler-agent
- name: load e2e test vm image
env:
TEST_IMAGE: ${{ needs.build-test-vm.outputs.vm-postgres-16-bullseye }}
timeout-minutes: 2
run: |
# Pull the docker image so we can re-tag it, because using a consistent tag inside the
# cluster means we can avoid dynamically editing the image used in the kuttl files.
docker pull "$TEST_IMAGE"
docker image tag "$TEST_IMAGE" "$IMG_E2E_TEST"
make load-example-vms
- run: make e2e
timeout-minutes: 15
- name: Get k8s logs and events
if: always()
run: |
if ! kubectl config current-context; then
echo "skipping cluster logs because no cluster found in kubectl context"
exit 0
fi
namespaces=$(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}')
for namespace in $namespaces; do
if [[ "$namespace" == "neonvm-system" ]] || [[ "$namespace" == kuttl-test-* ]]; then
tee_if_needed=$GITHUB_STEP_SUMMARY
else
tee_if_needed=/dev/null
fi
{
echo "<details>"
echo "<summary>Namespace=$namespace</summary>"
} | tee -a $tee_if_needed
pods=$(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}')
for pod in $pods; do
{
echo "<details>"
echo "<summary>- Namespace=$namespace Pod=$pod Logs</summary>"
echo "<pre>"
} | tee -a $tee_if_needed
restarts=$(
kubectl get pod -n $namespace $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' || echo '0'
)
{
if [ "$restarts" -ne 0 ]; then
echo "CONTAINER RESTARTED $restarts TIME(S)"
echo "Previous logs:"
kubectl logs -n $namespace -p $pod || echo 'Error getting logs'
echo "Current logs:"
kubectl logs -n $namespace $pod || echo 'Error getting logs'
else
echo "Logs:"
kubectl logs -n $namespace $pod || echo 'Error getting logs'
fi
} | tee -a $tee_if_needed
{
echo "</pre>"
echo "</details>"
} | tee -a $tee_if_needed
{
echo "<details>"
echo "<summary>- Namespace=$namespace Pod=$pod Events</summary>"
echo "<pre>"
} | tee -a $tee_if_needed
(kubectl get events --namespace $namespace --field-selector involvedObject.name=$pod || echo 'Error getting events') | tee -a $tee_if_needed
{
echo "</pre>"
echo "</pre>"
echo "</details>"
} | tee -a $tee_if_needed
done
echo "</details>" | tee -a $tee_if_needed
done
- name: Cleanup
if: always()
run: make ${{ matrix.cluster }}-destroy