Merge remote-tracking branch 'apache/main' into 4.19-vnf

agent/conf/agent.properties

@@ -279,6 +279,11 @@ hypervisor.type=kvm
 # If this parameter is used, property host.overcommit.mem.mb must be set to 0.
 #host.reserved.mem.mb=1024
 
+# Number of CPU cores to subtract from advertised available cores.
+# These are reserved for system activity, or otherwise share host CPU resources with
+# CloudStack VM allocation.
+# host.reserved.cpu.count = 0
+
 # The model of Watchdog timer to present to the Guest.
 # For all models refer to the libvirt documentation.
 #vm.watchdog.model=i6300esb

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -502,6 +502,15 @@ public class AgentProperties{
      */
     public static final Property<Integer> HOST_RESERVED_MEM_MB = new Property<>("host.reserved.mem.mb", 1024);
 
+    /**
+     * How many host CPUs to reserve for non-allocation.<br>
+     * This can be used to set aside CPU cores on the host for other tasks, such as running hyperconverged storage<br>
+     * processes, etc.
+     * Data type: Integer.<br>
+     * Default value: <code>0</code>
+     */
+    public static final Property<Integer> HOST_RESERVED_CPU_CORE_COUNT = new Property<>("host.reserved.cpu.count", 0);
+
     /**
      * The model of Watchdog timer to present to the Guest.<br>
      * For all models refer to the libvirt documentation.<br>

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtComputingResource.java

@@ -460,6 +460,8 @@ public class LibvirtComputingResource extends ServerResourceBase implements Serv
 
     private long dom0OvercommitMem;
 
+    private int dom0MinCpuCores;
+
     protected int cmdsTimeout;
     protected int stopTimeout;
     protected CPUStat cpuStat = new CPUStat();
@@ -1063,6 +1065,8 @@ public boolean configure(final String name, final Map<String, Object> params) th
         // Reserve 1GB unless admin overrides
         dom0MinMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_MEM_MB));
 
+        dom0MinCpuCores = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_RESERVED_CPU_CORE_COUNT);
+
         // Support overcommit memory for host if host uses ZSWAP, KSM and other memory
         // compressing technologies
         dom0OvercommitMem = ByteScaleUtils.mebibytesToBytes(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.HOST_OVERCOMMIT_MEM_MB));
@@ -3540,15 +3544,15 @@ private Map<String, String> getVersionStrings() {
     @Override
     public StartupCommand[] initialize() {
 
-        final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed);
+        final KVMHostInfo info = new KVMHostInfo(dom0MinMem, dom0OvercommitMem, manualCpuSpeed, dom0MinCpuCores);
 
         String capabilities = String.join(",", info.getCapabilities());
         if (dpdkSupport) {
             capabilities += ",dpdk";
         }
 
         final StartupRoutingCommand cmd =
-                new StartupRoutingCommand(info.getCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
+                new StartupRoutingCommand(info.getAllocatableCpus(), info.getCpuSpeed(), info.getTotalMemory(), info.getReservedMemory(), capabilities, hypervisorType,
                         RouterPrivateIpStrategy.HostLocal);
         cmd.setCpuSockets(info.getCpuSockets());
         fillNetworkInformation(cmd);

plugins/hypervisors/kvm/src/main/java/org/apache/cloudstack/utils/linux/KVMHostInfo.java

@@ -48,7 +48,8 @@ public class KVMHostInfo {
 
     private static final Logger LOGGER = Logger.getLogger(KVMHostInfo.class);
 
-    private int cpus;
+    private int totalCpus;
+    private int allocatableCpus;
     private int cpusockets;
     private long cpuSpeed;
     private long totalMemory;
@@ -58,16 +59,25 @@ public class KVMHostInfo {
 
     private static String cpuInfoFreqFileName = "/sys/devices/system/cpu/cpu0/cpufreq/base_frequency";
 
-    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed) {
+    public KVMHostInfo(long reservedMemory, long overCommitMemory, long manualSpeed, int reservedCpus) {
         this.cpuSpeed = manualSpeed;
         this.reservedMemory = reservedMemory;
         this.overCommitMemory = overCommitMemory;
         this.getHostInfoFromLibvirt();
         this.totalMemory = new MemStat(this.getReservedMemory(), this.getOverCommitMemory()).getTotal();
+        this.allocatableCpus = totalCpus - reservedCpus;
+        if (allocatableCpus < 1) {
+            LOGGER.warn(String.format("Aggressive reserved CPU config leaves no usable CPUs for VMs! Total system CPUs: %d, Reserved: %d, Allocatable: %d", totalCpus, reservedCpus, allocatableCpus));
+            allocatableCpus = 0;
+        }
+    }
+
+    public int getTotalCpus() {
+        return this.totalCpus;
     }
 
-    public int getCpus() {
-        return this.cpus;
+    public int getAllocatableCpus() {
+        return this.allocatableCpus;
     }
 
     public int getCpuSockets() {
@@ -189,7 +199,7 @@ private void getHostInfoFromLibvirt() {
             if (hosts.nodes > 0) {
                 this.cpusockets = hosts.sockets * hosts.nodes;
             }
-            this.cpus = hosts.cpus;
+            this.totalCpus = hosts.cpus;
 
             final LibvirtCapXMLParser parser = new LibvirtCapXMLParser();
             parser.parseCapabilitiesXML(capabilities);

plugins/hypervisors/kvm/src/test/java/org/apache/cloudstack/utils/linux/KVMHostInfoTest.java

@@ -73,8 +73,36 @@ public void manualCpuSpeedTest() throws Exception {
             Mockito.when(conn.close()).thenReturn(0);
             int manualSpeed = 500;
 
-            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed);
+            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, manualSpeed, 0);
             Assert.assertEquals(kvmHostInfo.getCpuSpeed(), manualSpeed);
         }
     }
+
+    @Test
+    public void reservedCpuCoresTest() throws Exception {
+        if (!System.getProperty("os.name").equals("Linux")) {
+            return;
+        }
+        try (MockedStatic<LibvirtConnection> ignored = Mockito.mockStatic(LibvirtConnection.class)) {
+            Connect conn = Mockito.mock(Connect.class);
+            NodeInfo nodeInfo = Mockito.mock(NodeInfo.class);
+            nodeInfo.cpus = 10;
+            String capabilitiesXml = "<capabilities></capabilities>";
+
+            Mockito.when(LibvirtConnection.getConnection()).thenReturn(conn);
+            Mockito.when(conn.nodeInfo()).thenReturn(nodeInfo);
+            Mockito.when(conn.getCapabilities()).thenReturn(capabilitiesXml);
+            Mockito.when(conn.close()).thenReturn(0);
+            int manualSpeed = 500;
+
+            KVMHostInfo kvmHostInfo = new KVMHostInfo(10, 10, 100, 2);
+            Assert.assertEquals("reserve two CPU cores", 8, kvmHostInfo.getAllocatableCpus());
+
+            kvmHostInfo = new KVMHostInfo(10, 10, 100, 0);
+            Assert.assertEquals("no reserve CPU core setting", 10, kvmHostInfo.getAllocatableCpus());
+
+            kvmHostInfo = new KVMHostInfo(10, 10, 100, 12);
+            Assert.assertEquals("Misconfigured/too large CPU reserve", 0, kvmHostInfo.getAllocatableCpus());
+        }
+    }
 }

server/src/main/java/org/apache/cloudstack/backup/BackupManagerImpl.java

@@ -1171,28 +1171,40 @@ protected void runInContext() {
                     }
 
                     final Map<VirtualMachine, Backup.Metric> metrics = backupProvider.getBackupMetrics(dataCenter.getId(), new ArrayList<>(vms));
-                    try {
-                        for (final VirtualMachine vm : metrics.keySet()) {
-                            final Backup.Metric metric = metrics.get(vm);
-                            if (metric != null) {
-                                // Sync out-of-band backups
-                                backupProvider.syncBackups(vm, metric);
-                                // Emit a usage event, update usage metric for the VM by the usage server
-                                UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(),
-                                        vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(),
-                                        vm.getBackupOfferingId(), null, metric.getBackupSize(), metric.getDataSize(),
-                                        Backup.class.getSimpleName(), vm.getUuid());
-                            }
-                        }
-                    } catch (final Throwable e) {
-                        LOG.error(String.format("Failed to sync backup usage metrics and out-of-band backups due to: [%s].", e.getMessage()), e);
-                    }
+                    syncBackupMetrics(backupProvider, metrics);
                 }
             } catch (final Throwable t) {
                 LOG.error(String.format("Error trying to run backup-sync background task due to: [%s].", t.getMessage()), t);
             }
         }
 
+        /**
+         * Tries to sync the VM backups. If one backup synchronization fails, only this VM backups are skipped, and the entire process does not stop.
+         */
+        private void syncBackupMetrics(final BackupProvider backupProvider, final Map<VirtualMachine, Backup.Metric> metrics) {
+            for (final VirtualMachine vm : metrics.keySet()) {
+                tryToSyncVMBackups(backupProvider, metrics, vm);
+            }
+        }
+
+        private void tryToSyncVMBackups(BackupProvider backupProvider, Map<VirtualMachine, Backup.Metric> metrics, VirtualMachine vm) {
+            try {
+                final Backup.Metric metric = metrics.get(vm);
+                if (metric != null) {
+                    LOG.debug(String.format("Trying to sync backups of VM [%s] using backup provider [%s].", vm.getUuid(), backupProvider.getName()));
+                    // Sync out-of-band backups
+                    backupProvider.syncBackups(vm, metric);
+                    // Emit a usage event, update usage metric for the VM by the usage server
+                    UsageEventUtils.publishUsageEvent(EventTypes.EVENT_VM_BACKUP_USAGE_METRIC, vm.getAccountId(),
+                            vm.getDataCenterId(), vm.getId(), "Backup-" + vm.getHostName() + "-" + vm.getUuid(),
+                            vm.getBackupOfferingId(), null, metric.getBackupSize(), metric.getDataSize(),
+                            Backup.class.getSimpleName(), vm.getUuid());
+                }
+            } catch (final Exception e) {
+                LOG.error(String.format("Failed to sync backup usage metrics and out-of-band backups of VM [%s] due to: [%s].", vm.getUuid(), e.getMessage()), e);
+            }
+        }
+
         @Override
         public Long getDelay() {
             return BackupSyncPollingInterval.value() * 1000L;

test/integration/smoke/test_vm_schedule.py

@@ -596,6 +596,10 @@ def test_05_vmschedule_test_e2e(self):
             time.sleep(30)
             current_state = self.virtual_machine.update(self.apiclient).state
             if previous_state != current_state:
+                # Add these checks because VMs can take some time to start or stop
+                if (previous_state == 'Starting' and current_state in ('Starting', 'Running')) or (
+                        previous_state == 'Stopping' and current_state in ('Stopping', 'Stopped')):
+                    continue
                 self.debug(
                     "VM changed state from %s to %s" % (previous_state, current_state)
                 )