Merge remote-tracking branch 'apache/main' into 4.20-dynamic-routing

.asf.yaml

@@ -59,6 +59,7 @@ github:
     - hsato03
     - bernardodemarco
     - abh1sar
+    - FelipeM525
 
   protected_branches: ~
 

core/src/test/java/com/cloud/serializer/GsonHelperTest.java

@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package com.cloud.serializer;
+
+import com.cloud.agent.api.to.NfsTO;
+import com.cloud.storage.DataStoreRole;
+import com.google.gson.Gson;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test cases to verify working order of GsonHelper.java
+ * with regards to a concrete implementation of the DataStoreTO
+ * interface
+ */
+public class GsonHelperTest {
+
+    private Gson gson;
+    private Gson gsonLogger;
+    private NfsTO nfsTO;
+
+    @Before
+    public void setUp() {
+        gson = GsonHelper.getGson();
+        gsonLogger = GsonHelper.getGsonLogger();
+        nfsTO = new NfsTO("http://example.com", DataStoreRole.Primary);
+    }
+
+    @Test
+    public void testGsonSerialization() {
+        String json = gson.toJson(nfsTO);
+        assertNotNull(json);
+        assertTrue(json.contains("\"_url\":\"http://example.com\""));
+        assertTrue(json.contains("\"_role\":\"Primary\""));
+    }
+
+    @Test
+    public void testGsonDeserialization() {
+        String json = "{\"_url\":\"http://example.com\",\"_role\":\"Primary\"}";
+        NfsTO deserializedNfsTO = gson.fromJson(json, NfsTO.class);
+        assertNotNull(deserializedNfsTO);
+        assertEquals("http://example.com", deserializedNfsTO.getUrl());
+        assertEquals(DataStoreRole.Primary, deserializedNfsTO.getRole());
+    }
+
+    @Test
+    public void testGsonLoggerSerialization() {
+        String json = gsonLogger.toJson(nfsTO);
+        assertNotNull(json);
+        assertTrue(json.contains("\"_url\":\"http://example.com\""));
+        assertTrue(json.contains("\"_role\":\"Primary\""));
+    }
+
+    @Test
+    public void testGsonLoggerDeserialization() {
+        String json ="{\"_url\":\"http://example.com\",\"_role\":\"Primary\"}";
+        NfsTO deserializedNfsTO = gsonLogger.fromJson(json, NfsTO.class);
+        assertNotNull(deserializedNfsTO);
+        assertEquals("http://example.com", deserializedNfsTO.getUrl());
+        assertEquals(DataStoreRole.Primary, deserializedNfsTO.getRole());
+    }
+}

core/src/test/java/com/cloud/storage/template/OVAProcessorTest.java

@@ -131,5 +131,25 @@ public void testGetVirtualSize() throws Exception {
         Assert.assertEquals(virtualSize, processor.getVirtualSize(mockFile));
         Mockito.verify(mockFile, Mockito.times(0)).length();
     }
+    @Test
+    public void testProcessWithLargeFileSize() throws Exception {
+        String templatePath = "/tmp";
+        String templateName = "large_template";
+        long virtualSize = 10_000_000_000L;
+        long actualSize = 5_000_000_000L;
+
+        Mockito.when(mockStorageLayer.exists(Mockito.anyString())).thenReturn(true);
+        Mockito.when(mockStorageLayer.getSize(Mockito.anyString())).thenReturn(actualSize);
+        Mockito.doReturn(virtualSize).when(processor).getTemplateVirtualSize(Mockito.anyString(), Mockito.anyString());
 
+        try (MockedConstruction<Script> ignored = Mockito.mockConstruction(Script.class, (mock, context) -> {
+            Mockito.when(mock.execute()).thenReturn(null);
+        })) {
+            Processor.FormatInfo info = processor.process(templatePath, null, templateName);
+            Assert.assertEquals(Storage.ImageFormat.OVA, info.format);
+            Assert.assertEquals("actual size:", actualSize, info.size);
+            Assert.assertEquals("virtual size:", virtualSize, info.virtualSize);
+            Assert.assertEquals("template name:", templateName + ".ova", info.filename);
+        }
+    }
 }

deps/install-non-oss.sh

@@ -40,8 +40,50 @@ mvn install:install-file -Dfile=vim25_65.jar        -DgroupId=com.cloud.com.vmwa
 # From https://my.vmware.com/group/vmware/details?downloadGroup=WEBCLIENTSDK67U2&productId=742
 mvn install:install-file -Dfile=vim25_67.jar        -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25    -Dversion=6.7   -Dpackaging=jar
 
+# https://my.vmware.com/group/vmware/downloads/get-download?downloadGroup=VS-MGMT-SDK700
+mvn install:install-file -Dfile=vim25_70.jar  -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25   -Dversion=7.0   -Dpackaging=jar
+
+# https://my.vmware.com/group/vmware/downloads/get-download?downloadGroup=VS-MGMT-SDK80
+mvn install:install-file -Dfile=vim25_80.jar  -DgroupId=com.cloud.com.vmware -DartifactId=vmware-vim25   -Dversion=8.0   -Dpackaging=jar
+
 # From https://my.vmware.com/group/vmware/get-download?downloadGroup=VS-MGMT-SDK65
 mvn install:install-file -Dfile=pbm_65.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.5 -Dpackaging=jar
 
 # From https://my.vmware.com/group/vmware/downloads/get-download?downloadGroup=VS-MGMT-SDK67
 mvn install:install-file -Dfile=pbm_67.jar -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm -Dversion=6.7 -Dpackaging=jar
+
+# https://my.vmware.com/group/vmware/downloads/get-download?downloadGroup=VS-MGMT-SDK700
+mvn install:install-file -Dfile=pbm_70.jar    -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm     -Dversion=7.0   -Dpackaging=jar
+
+# https://my.vmware.com/group/vmware/downloads/get-download?downloadGroup=VS-MGMT-SDK80
+mvn install:install-file -Dfile=pbm_80.jar    -DgroupId=com.cloud.com.vmware -DartifactId=vmware-pbm     -Dversion=8.0   -Dpackaging=jar
+
+# From https://github.com/vmware/vsphere-automation-sdk-java/tree/master/lib
+mvn install:install-file -Dfile=vapi-runtime-2.15.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-runtime -Dversion=2.15.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vapi-runtime-2.37.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-runtime -Dversion=2.37.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vapi-runtime-2.40.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-runtime -Dversion=2.40.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vapi-authentication-2.15.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-authentication -Dversion=2.15.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vapi-authentication-2.37.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-authentication -Dversion=2.37.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vapi-authentication-2.40.0.jar -DgroupId=com.vmware.vapi -DartifactId=vapi-authentication -Dversion=2.40.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vsphereautomation-client-sdk-3.3.0.jar -DgroupId=com.vmware.vsphereautomation.client -DartifactId=vsphereautomation-client-sdk -Dversion=3.3.0 -Dpackaging=jar
+mvn install:install-file -Dfile=vsphereautomation-client-sdk-4.0.0.jar -DgroupId=com.vmware.vsphereautomation.client -DartifactId=vsphereautomation-client-sdk -Dversion=4.0.0 -Dpackaging=jar
+
+# From https://customerconnect.vmware.com/downloads/details?downloadGroup=NSX-4011-SDK-JAVA&productId=1324
+mvn install:install-file -Dfile=nsx-java-sdk-4.1.0.2.0.jar -DgroupId=com.vmware -DartifactId=nsx-java-sdk -Dversion=4.1.0.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile=nsx-gpm-java-sdk-4.1.0.2.0.jar -DgroupId=com.vmware -DartifactId=nsx-gpm-java-sdk -Dversion=4.1.0.2.0 -Dpackaging=jar
+mvn install:install-file -Dfile=nsx-policy-java-sdk-4.1.0.2.0.jar -DgroupId=com.vmware -DartifactId=nsx-policy-java-sdk -Dversion=4.1.0.2.0 -Dpackaging=jar
+
+# From http://support.netapp.com/  (not available online, contact your support representative)
+# Version: 4.0 (http://community.netapp.com/t5/Developer-Network-Articles-and-Resources/NetApp-Manageability-NM-SDK-Introduction-and-Download-Information/ta-p/86418)
+if [ -e cloud-manageontap.jar ]; then mv cloud-manageontap.jar manageontap.jar;  fi
+mvn install:install-file -Dfile=manageontap.jar     -DgroupId=com.cloud.com.netapp -DartifactId=manageontap     -Dversion=4.0   -Dpackaging=jar
+
+# From https://juniper.github.io/contrail-maven/snapshots/net/juniper/contrail/juniper-contrail-api/1.0-SNAPSHOT/juniper-contrail-api-1.0-20131001.003401-3.jar
+# New version can be found at https://github.com/Juniper/contrail-maven/tree/master/releases/net/juniper/contrail/juniper-contrail-api/1.2
+mvn install:install-file -Dfile=juniper-contrail-api-1.0-SNAPSHOT.jar -DgroupId=net.juniper.contrail -DartifactId=juniper-contrail-api -Dversion=1.0-SNAPSHOT -Dpackaging=jar
+
+# From https://github.com/radu-todirica/tungsten-api/raw/master/net/juniper/tungsten/juniper-tungsten-api/2.0/juniper-tungsten-api-2.0.jar
+mvn install:install-file -Dfile=juniper-tungsten-api-2.0.jar -DgroupId=net.juniper.tungsten -DartifactId=juniper-tungsten-api -Dversion=2.0 -Dpackaging=jar
+
+exit 0
+

engine/api/src/main/java/com/cloud/vm/VirtualMachineManager.java

@@ -96,7 +96,7 @@ public interface VirtualMachineManager extends Manager {
     ConfigKey<String> VmMetadataProductName = new ConfigKey<>("Advanced", String.class,
             "vm.metadata.product", "",
             "If provided, a custom product name will be used in the instance metadata. When an empty" +
-                    "value is set then default product name will be 'CloudStack <HYPERVISIOR_NAME> Hypervisor'. " +
+                    "value is set then default product name will be 'CloudStack <HYPERVISOR_NAME> Hypervisor'. " +
                     "A custom product name may break cloud-init functionality with CloudStack datasource. Please " +
                     "refer documentation",
             true, ConfigKey.Scope.Zone);

engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java

@@ -1108,7 +1108,7 @@ protected void updateVmMetadataManufacturerAndProduct(VirtualMachineTO vmTO, VMI
         }
         vmTO.setMetadataManufacturer(metadataManufacturer);
         String metadataProduct = VmMetadataProductName.valueIn(vm.getDataCenterId());
-        if (StringUtils.isBlank(metadataManufacturer)) {
+        if (StringUtils.isBlank(metadataProduct)) {
             metadataProduct = String.format("CloudStack %s Hypervisor", vm.getHypervisorType().toString());
         }
         vmTO.setMetadataProductName(metadataProduct);

engine/orchestration/src/test/java/com/cloud/vm/VirtualMachineManagerImplTest.java

@@ -1272,6 +1272,7 @@ private Pair<VirtualMachineTO, VMInstanceVO> getDummyVmTOAndVm() {
                 false, false, "Pass");
         VMInstanceVO vm = Mockito.mock(VMInstanceVO.class);
         Mockito.when(vm.getDataCenterId()).thenReturn(1L);
+        Mockito.when(vm.getHypervisorType()).thenReturn(HypervisorType.KVM);
         return new Pair<>(virtualMachineTO, vm);
     }
 
@@ -1284,6 +1285,17 @@ public void testUpdateVmMetadataManufacturerAndProductDefaultManufacturer() {
         Assert.assertEquals(VirtualMachineManager.VmMetadataManufacturer.defaultValue(), to.getMetadataManufacturer());
     }
 
+    @Test
+    public void testUpdateVmMetadataManufacturerAndProductCustomManufacturerDefaultProduct() {
+        String manufacturer = "Custom";
+        overrideVmMetadataConfigValue(manufacturer, "");
+        Pair<VirtualMachineTO, VMInstanceVO> pair = getDummyVmTOAndVm();
+        VirtualMachineTO to = pair.first();
+        virtualMachineManagerImpl.updateVmMetadataManufacturerAndProduct(to, pair.second());
+        Assert.assertEquals(manufacturer, to.getMetadataManufacturer());
+        Assert.assertEquals("CloudStack KVM Hypervisor", to.getMetadataProductName());
+    }
+
     @Test
     public void testUpdateVmMetadataManufacturerAndProductCustomManufacturer() {
         String manufacturer = UUID.randomUUID().toString();

framework/db/src/main/java/com/cloud/utils/db/Filter.java

@@ -22,6 +22,7 @@
 
 import com.cloud.utils.Pair;
 import com.cloud.utils.ReflectUtil;
+import org.apache.commons.lang3.StringUtils;
 
 /**
  *  Try to use static initialization to help you in finding incorrect
@@ -59,6 +60,11 @@ public Filter(long limit) {
         _orderBy = " ORDER BY RAND() LIMIT " + limit;
     }
 
+    public Filter(Long offset, Long limit) {
+        _offset = offset;
+        _limit = limit;
+    }
+
     /**
      * Note that this copy constructor does not copy offset and limit.
      * @param that filter
@@ -70,6 +76,10 @@ public Filter(Filter that) {
     }
 
     public void addOrderBy(Class<?> clazz, String field, boolean ascending) {
+        addOrderBy(clazz, field, ascending, null);
+    }
+
+    public void addOrderBy(Class<?> clazz, String field, boolean ascending, String tableAlias) {
         if (field == null) {
             return;
         }
@@ -83,7 +93,9 @@ public void addOrderBy(Class<?> clazz, String field, boolean ascending) {
         String name = column != null ? column.name() : field;
 
         StringBuilder order = new StringBuilder();
-        if (column == null || column.table() == null || column.table().length() == 0) {
+        if (StringUtils.isNotBlank(tableAlias)) {
+            order.append(tableAlias);
+        } else if (column == null || column.table() == null || column.table().length() == 0) {
             order.append(DbUtil.getTableName(clazz));
         } else {
             order.append(column.table());

plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtVMDef.java

@@ -127,6 +127,9 @@ public GuestType getGuestType() {
         }
 
         public String getManufacturer() {
+            if (StringUtils.isEmpty(manufacturer)) {
+                return "Apache Software Foundation";
+            }
             return manufacturer;
         }
 
@@ -135,6 +138,9 @@ public void setManufacturer(String manufacturer) {
         }
 
         public String getProduct() {
+            if (StringUtils.isEmpty(product)) {
+                return "CloudStack KVM Hypervisor";
+            }
             return product;
         }
 

server/src/main/java/com/cloud/resource/ResourceManagerImpl.java

@@ -3400,7 +3400,8 @@ public boolean isHostGpuEnabled(final long hostId) {
 
     @Override
     public List<HostGpuGroupsVO> listAvailableGPUDevice(final long hostId, final String groupName, final String vgpuType) {
-        final Filter searchFilter = new Filter(VGPUTypesVO.class, "remainingCapacity", false, null, null);
+        Filter searchFilter = new Filter(null, null);
+        searchFilter.addOrderBy(VGPUTypesVO.class, "remainingCapacity", false, "groupId");
         final SearchCriteria<HostGpuGroupsVO> sc = _gpuAvailability.create();
         sc.setParameters("hostId", hostId);
         sc.setParameters("groupName", groupName);

server/src/test/java/com/cloud/user/PasswordPolicyImplTest.java

@@ -160,4 +160,22 @@ public void validateRegexTestWithoutRegex() {
         passwordPolicySpy.validateIfPasswordMatchesRegex("abcd123", "user", null);
     }
 
+    @Test
+    public void validateCombinationOfPolicies() {
+        Mockito.doReturn(2).when(passwordPolicySpy).getPasswordPolicyMinimumSpecialCharacters(null);
+        Mockito.doReturn(1).when(passwordPolicySpy).getPasswordPolicyMinimumUpperCaseLetters(null);
+        Mockito.doReturn(1).when(passwordPolicySpy).getPasswordPolicyMinimumLowerCaseLetters(null);
+        Mockito.doReturn(1).when(passwordPolicySpy).getPasswordPolicyMinimumDigits(null);
+        Mockito.doReturn(8).when(passwordPolicySpy).getPasswordPolicyMinimumLength(null);
+        Mockito.doReturn(false).when(passwordPolicySpy).getPasswordPolicyAllowPasswordToContainUsername(null);
+
+        String password = "Ab1!@#cd";
+        passwordPolicySpy.validateIfPasswordContainsTheMinimumNumberOfSpecialCharacters(2, password, null);
+        passwordPolicySpy.validateIfPasswordContainsTheMinimumNumberOfUpperCaseLetters(1, password, null);
+        passwordPolicySpy.validateIfPasswordContainsTheMinimumNumberOfLowerCaseLetters(1, password, null);
+        passwordPolicySpy.validateIfPasswordContainsTheMinimumNumberOfDigits(1, password, null);
+        passwordPolicySpy.validateIfPasswordContainsTheMinimumLength(password, "user", null);
+        passwordPolicySpy.validateIfPasswordContainsTheUsername(password, "user", null);
+    }
+
 }

systemvm/debian/opt/cloud/bin/setup/init.sh

@@ -112,6 +112,8 @@ config_guest() {
           fi
           ;;
      vmware)
+          # explicitly update the module dependencies
+          depmod -a
           # system time sync'd with host via vmware tools
           systemctl stop ntpd
           systemctl disable ntpd

tools/appliance/build.sh

@@ -235,7 +235,7 @@ function stage_vmx() {
 displayname = "${1}"
 annotation = "${1}"
 guestos = "otherlinux-64"
-virtualHW.version = "11"
+virtualHW.version = "13"
 config.version = "8"
 numvcpus = "1"
 cpuid.coresPerSocket = "1"

tools/appliance/systemvmtemplate/scripts/configure_grub.sh

@@ -34,7 +34,7 @@ function configure_grub() {
 GRUB_DEFAULT=0
 GRUB_TIMEOUT=0
 GRUB_DISTRIBUTOR=Debian
-GRUB_CMDLINE_LINUX_DEFAULT="quiet"
+GRUB_CMDLINE_LINUX_DEFAULT="quiet fsck.mode=force fsck.repair=yes"
 GRUB_CMDLINE_LINUX="console=tty0 console=ttyS0,115200n8 console=hvc0 earlyprintk=xen net.ifnames=0 biosdevname=0 debian-installer=en_US nomodeset"
 GRUB_CMDLINE_XEN="com1=115200 console=com1"
 GRUB_TERMINAL="console serial"

tools/appliance/systemvmtemplate/scripts/configure_systemvm_services.sh

@@ -41,7 +41,7 @@ function configure_issue() {
 
    __?.o/  Apache CloudStack SystemVM $CLOUDSTACK_RELEASE
   (  )#    https://cloudstack.apache.org
- (___(_)   Debian GNU/Linux 11 \n \l
+ (___(_)   Debian GNU/Linux 12 \n \l
 
 EOF
 }

tools/appliance/systemvmtemplate/template-base_aarch64-target_aarch64.json

@@ -32,8 +32,8 @@
       "format": "qcow2",
       "headless": true,
       "http_directory": "http",
-      "iso_checksum": "sha512:14c2ca243ee7f6e447cc4466296d974ee36645c06d72043236c3fbea78f1948d3af88d65139105a475288f270e4b636e6885143d01bdf69462620d1825e470ae",
-      "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/arm64/iso-cd/debian-12.5.0-arm64-netinst.iso",
+      "iso_checksum": "sha512:fc3560bb586af14b1d77ab7c2806616916926afcbd5cb3fd5a04a5633dfd91cfbbccada1a123f1ea14c480153b731cbee72a230cea17fd9116b9df8444d8df1c",
+      "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/arm64/iso-cd/debian-12.7.0-arm64-netinst.iso",
       "net_device": "virtio-net",
       "output_directory": "../dist",
       "qemu_binary": "qemu-system-aarch64",

tools/appliance/systemvmtemplate/template-base_x86_64-target_aarch64.json

@@ -31,8 +31,8 @@
       "format": "qcow2",
       "headless": true,
       "http_directory": "http",
-      "iso_checksum": "sha512:14c2ca243ee7f6e447cc4466296d974ee36645c06d72043236c3fbea78f1948d3af88d65139105a475288f270e4b636e6885143d01bdf69462620d1825e470ae",
-      "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/arm64/iso-cd/debian-12.5.0-arm64-netinst.iso",
+      "iso_checksum": "sha512:fc3560bb586af14b1d77ab7c2806616916926afcbd5cb3fd5a04a5633dfd91cfbbccada1a123f1ea14c480153b731cbee72a230cea17fd9116b9df8444d8df1c",
+      "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/arm64/iso-cd/debian-12.7.0-arm64-netinst.iso",
       "net_device": "virtio-net",
       "output_directory": "../dist",
       "qemu_binary": "qemu-system-aarch64",

tools/appliance/systemvmtemplate/template-base_x86_64-target_x86_64.json

@@ -27,8 +27,8 @@
       "format": "qcow2",
       "headless": true,
       "http_directory": "http",
-      "iso_checksum": "sha512:33c08e56c83d13007e4a5511b9bf2c4926c4aa12fd5dd56d493c0653aecbab380988c5bf1671dbaea75c582827797d98c4a611f7fb2b131fbde2c677d5258ec9",
-      "iso_url": "https://cdimage.debian.org/mirror/cdimage/archive/12.5.0/amd64/iso-cd/debian-12.5.0-amd64-netinst.iso",
+      "iso_checksum": "sha512:e0bd9ba03084a6fd42413b425a2d20e3731678a31fe5fb2cc84f79332129afca2ad4ec897b4224d6a833afaf28a5d938b0fe5d680983182944162c6825b135ce",
+      "iso_url": "https://cdimage.debian.org/mirror/cdimage/release/12.7.0/amd64/iso-cd/debian-12.7.0-amd64-netinst.iso",
       "net_device": "virtio-net",
       "output_directory": "../dist",
       "qemuargs": [