neonvmrunner,neonvmcontroller: allocate all possible CPUs to the VM a…

…nd mark vmSpec.Guest.CPUs.Min as online during the boot

Signed-off-by: Misha Sakhnov <[email protected]>

neonvm-controller/cmd/main.go

@@ -95,6 +95,7 @@ func main() {
 	var concurrencyLimit int
 	var skipUpdateValidationFor map[types.NamespacedName]struct{}
 	var disableRunnerCgroup bool
+	var useOnlineOfflining bool
 	var qemuDiskCacheSettings string
 	var defaultMemoryProvider vmv1.MemoryProvider
 	var memhpAutoMovableRatio string
@@ -105,6 +106,7 @@ func main() {
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
+	flag.BoolVar(&useOnlineOfflining, "use-online-offlining", false, "Use online offlining for CPU scaling")
 	flag.IntVar(&concurrencyLimit, "concurrency-limit", 1, "Maximum number of concurrent reconcile operations")
 	flag.Func(
 		"skip-update-validation-for",
@@ -187,7 +189,7 @@ func main() {
 	reconcilerMetrics := controllers.MakeReconcilerMetrics()
 
 	rc := &controllers.ReconcilerConfig{
-		DisableRunnerCgroup:     disableRunnerCgroup,
+		UseOnlineOfflining:      useOnlineOfflining,
 		MaxConcurrentReconciles: concurrencyLimit,
 		SkipUpdateValidationFor: skipUpdateValidationFor,
 		QEMUDiskCacheSettings:   qemuDiskCacheSettings,

neonvm-controller/deployment.yaml

@@ -55,6 +55,7 @@ spec:
         - "--concurrency-limit=128"
         - "--skip-update-validation-for="
         - "--disable-runner-cgroup"
+        - "--use-online-offlining"        
         # See #775 and its links.
         # * cache.writeback=on - don't set O_DSYNC (don't flush every write)
         # * cache.direct=on    - use O_DIRECT (don't abuse host's page cache!)

neonvm-runner/cmd/main.go

@@ -616,6 +616,7 @@ type Config struct {
 	diskCacheSettings    string
 	memoryProvider       vmv1.MemoryProvider
 	autoMovableRatio     string
+	useOnlineOfflining   bool
 }
 
 func newConfig(logger *zap.Logger) *Config {
@@ -629,6 +630,7 @@ func newConfig(logger *zap.Logger) *Config {
 		diskCacheSettings:    "cache=none",
 		memoryProvider:       "", // Require that this is explicitly set. We'll check later.
 		autoMovableRatio:     "", // Require that this is explicitly set IFF memoryProvider is VirtioMem. We'll check later.
+		useOnlineOfflining:   false,
 	}
 	flag.StringVar(&cfg.vmSpecDump, "vmspec", cfg.vmSpecDump,
 		"Base64 encoded VirtualMachine json specification")
@@ -649,7 +651,7 @@ func newConfig(logger *zap.Logger) *Config {
 	flag.Func("memory-provider", "Set provider for memory hotplug", cfg.memoryProvider.FlagFunc)
 	flag.StringVar(&cfg.autoMovableRatio, "memhp-auto-movable-ratio",
 		cfg.autoMovableRatio, "Set value of kernel's memory_hotplug.auto_movable_ratio [virtio-mem only]")
-
+	flag.BoolVar(&cfg.useOnlineOfflining, "use-online-offlining", false, "Use online offlining for CPU scaling")
 	flag.Parse()
 
 	if cfg.memoryProvider == "" {
@@ -759,7 +761,7 @@ func run(logger *zap.Logger) error {
 
 	tg.Go("qemu-cmd", func(logger *zap.Logger) error {
 		var err error
-		qemuCmd, err = buildQEMUCmd(cfg, logger, vmSpec, &vmStatus, enableSSH, swapSize, hostname)
+		qemuCmd, err = buildQEMUCmd(cfg, logger, vmSpec, &vmStatus, enableSSH, swapSize, hostname, cfg.useOnlineOfflining)
 		return err
 	})
 
@@ -813,6 +815,7 @@ func buildQEMUCmd(
 	enableSSH bool,
 	swapSize *resource.Quantity,
 	hostname string,
+	useOnlineOfflining bool,
 ) ([]string, error) {
 	// prepare qemu command line
 	qemuCmd := []string{
@@ -890,12 +893,23 @@ func buildQEMUCmd(
 		logger.Warn("not using KVM acceleration")
 	}
 	qemuCmd = append(qemuCmd, "-cpu", "max")
-	qemuCmd = append(qemuCmd, "-smp", fmt.Sprintf(
-		"cpus=%d,maxcpus=%d,sockets=1,cores=%d,threads=1",
-		vmSpec.Guest.CPUs.Min.RoundedUp(),
-		vmSpec.Guest.CPUs.Max.RoundedUp(),
-		vmSpec.Guest.CPUs.Max.RoundedUp(),
-	))
+	if useOnlineOfflining {
+		// if we use online offlining we specify start cpus equal to max cpus
+		qemuCmd = append(qemuCmd, "-smp", fmt.Sprintf(
+			"cpus=%d,maxcpus=%d,sockets=1,cores=%d,threads=1",
+			vmSpec.Guest.CPUs.Max.RoundedUp(),
+			vmSpec.Guest.CPUs.Max.RoundedUp(),
+			vmSpec.Guest.CPUs.Max.RoundedUp(),
+		))
+	} else {
+		// if we use hotplug we specify start cpus equal to min cpus and scale using udev rules for cpu plug events
+		qemuCmd = append(qemuCmd, "-smp", fmt.Sprintf(
+			"cpus=%d,maxcpus=%d,sockets=1,cores=%d,threads=1",
+			vmSpec.Guest.CPUs.Min.RoundedUp(),
+			vmSpec.Guest.CPUs.Max.RoundedUp(),
+			vmSpec.Guest.CPUs.Max.RoundedUp(),
+		))
+	}
 
 	// memory details
 	logger.Info(fmt.Sprintf("Using memory provider %s", cfg.memoryProvider))
@@ -983,6 +997,10 @@ func makeKernelCmdline(cfg *Config, vmSpec *vmv1.VirtualMachineSpec, vmStatus *v
 	if cfg.appendKernelCmdline != "" {
 		cmdlineParts = append(cmdlineParts, cfg.appendKernelCmdline)
 	}
+	if cfg.useOnlineOfflining {
+		// if we use online offlining we need to specify the start cpus as min CPUs
+		cmdlineParts = append(cmdlineParts, fmt.Sprintf("maxcpus=%d", vmSpec.Guest.CPUs.Min.RoundedUp()))
+	}
 
 	return strings.Join(cmdlineParts, " ")
 }

pkg/neonvm/controllers/config.go

@@ -11,6 +11,11 @@ import (
 // ReconcilerConfig stores shared configuration for VirtualMachineReconciler and
 // VirtualMachineMigrationReconciler.
 type ReconcilerConfig struct {
+	// UseOnlineOfflining, if true, enables using online offlining for new VM runner pods instead of QMP cpu hotplugging.
+	//
+	// This is defined as a config option so we can do a gradual rollout of this change.
+	UseOnlineOfflining bool
+
 	// DisableRunnerCgroup, if true, disables running QEMU in a cgroup in new VM runner pods.
 	// Fractional CPU scaling will continue to *pretend* to work, but it will not do anything in
 	// practice.

pkg/neonvm/controllers/vm_controller.go

@@ -1447,6 +1447,9 @@ func podSpec(
 					}},
 					Command: func() []string {
 						cmd := []string{"runner"}
+						if config.UseOnlineOfflining {
+							cmd = append(cmd, "-use-online-offlining") // TODO: need to force enable-dummy-cpu-server
+						}
 						if config.DisableRunnerCgroup {
 							cmd = append(cmd, "-skip-cgroup-management")
 							// cgroup management disabled, but we still need something to provide

vm-deploy.yaml

@@ -14,7 +14,7 @@ spec:
   schedulerName: autoscale-scheduler
   enableSSH: true
   guest:
-    cpus: { min: 0.25, use: 0.25, max: 1.25 }
+    cpus: { min: 0.25, use: 0.25, max: 3 }
     memorySlotSize: 1Gi
     memorySlots: { min: 1, use: 1, max: 5 }
     rootDisk: