Merge branch '4.19' into 4.19-fix-high-cpu-agent-restart

PULL_REQUEST_TEMPLATE.md

@@ -23,6 +23,7 @@ This PR...
 - [ ] Enhancement (improves an existing feature and functionality)
 - [ ] Cleanup (Code refactoring and cleanup, that may add test cases)
 - [ ] build/CI
+- [ ] test (unit or integration test code)
 
 ### Feature/Enhancement Scale or Bug Severity
 

api/src/main/java/com/cloud/network/VpcVirtualNetworkApplianceService.java

@@ -29,7 +29,6 @@ public interface VpcVirtualNetworkApplianceService extends VirtualNetworkApplian
     /**
      * @param router
      * @param network
-     * @param isRedundant
      * @param params TODO
      * @return
      * @throws ConcurrentOperationException
@@ -42,11 +41,30 @@ boolean addVpcRouterToGuestNetwork(VirtualRouter router, Network network, Map<Vi
     /**
      * @param router
      * @param network
-     * @param isRedundant
      * @return
      * @throws ConcurrentOperationException
      * @throws ResourceUnavailableException
      */
     boolean removeVpcRouterFromGuestNetwork(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
 
+
+    /**
+     * @param router
+     * @param network
+     * @return
+     * @throws ConcurrentOperationException
+     * @throws ResourceUnavailableException
+     */
+    boolean stopKeepAlivedOnRouter(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
+
+
+    /**
+     * @param router
+     * @param network
+     * @return
+     * @throws ConcurrentOperationException
+     * @throws ResourceUnavailableException
+     */
+    boolean startKeepAlivedOnRouter(VirtualRouter router, Network network) throws ConcurrentOperationException, ResourceUnavailableException;
+
 }

api/src/main/java/com/cloud/storage/StorageService.java

@@ -29,6 +29,7 @@
 import org.apache.cloudstack.api.command.admin.storage.DeleteSecondaryStagingStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.SyncStoragePoolCmd;
 import org.apache.cloudstack.api.command.admin.storage.UpdateObjectStoragePoolCmd;
+import org.apache.cloudstack.api.command.admin.storage.UpdateImageStoreCmd;
 import org.apache.cloudstack.api.command.admin.storage.UpdateStoragePoolCmd;
 
 import com.cloud.exception.DiscoveryException;
@@ -110,6 +111,8 @@ public interface StorageService {
      */
     ImageStore migrateToObjectStore(String name, String url, String providerName, Map<String, String> details) throws DiscoveryException;
 
+    ImageStore updateImageStore(UpdateImageStoreCmd cmd);
+
     ImageStore updateImageStoreStatus(Long id, Boolean readonly);
 
     void updateStorageCapabilities(Long poolId, boolean failOnChecks);

api/src/main/java/org/apache/cloudstack/api/command/admin/storage/UpdateImageStoreCmd.java

@@ -41,10 +41,17 @@ public class UpdateImageStoreCmd extends BaseCmd {
     @Parameter(name = ApiConstants.ID, type = CommandType.UUID, entityType = ImageStoreResponse.class, required = true, description = "Image Store UUID")
     private Long id;
 
-    @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, required = true, description = "If set to true, it designates the corresponding image store to read-only, " +
-            "hence not considering them during storage migration")
+    @Parameter(name = ApiConstants.NAME, type = CommandType.STRING, required = false, description = "The new name for the Image Store.")
+    private String name;
+
+    @Parameter(name = ApiConstants.READ_ONLY, type = CommandType.BOOLEAN, required = false,
+            description = "If set to true, it designates the corresponding image store to read-only, hence not considering them during storage migration")
     private Boolean readonly;
 
+    @Parameter(name = ApiConstants.CAPACITY_BYTES, type = CommandType.LONG, required = false,
+            description = "The number of bytes CloudStack can use on this image storage.\n\tNOTE: this will be overwritten by the StatsCollector as soon as there is a SSVM to query the storage.")
+    private Long capacityBytes;
+
     /////////////////////////////////////////////////////
     /////////////////// Accessors ///////////////////////
     /////////////////////////////////////////////////////
@@ -53,17 +60,25 @@ public Long getId() {
         return id;
     }
 
+    public String getName() {
+        return name;
+    }
+
     public Boolean getReadonly() {
         return readonly;
     }
 
+    public Long getCapacityBytes() {
+        return capacityBytes;
+    }
+
     /////////////////////////////////////////////////////
     /////////////// API Implementation///////////////////
     /////////////////////////////////////////////////////
 
     @Override
     public void execute() {
-        ImageStore result = _storageService.updateImageStoreStatus(getId(), getReadonly());
+        ImageStore result = _storageService.updateImageStore(this);
         ImageStoreResponse storeResponse = null;
         if (result != null) {
             storeResponse = _responseGenerator.createImageStoreResponse(result);

api/src/main/java/org/apache/cloudstack/api/command/admin/vm/ListVnfAppliancesCmdByAdmin.java

@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.admin.vm;
+
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.admin.AdminCmd;
+
+import org.apache.cloudstack.api.command.user.vm.ListVnfAppliancesCmd;
+import org.apache.cloudstack.api.response.UserVmResponse;
+
+@APICommand(name = "listVnfAppliances", description = "List VNF appliance owned by the account.",
+        responseObject = UserVmResponse.class,
+        responseView = ResponseView.Full,
+        entityType = {VirtualMachine.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin},
+        since = "4.19.1")
+public class ListVnfAppliancesCmdByAdmin extends ListVnfAppliancesCmd implements AdminCmd {
+}

api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/CreateAutoScaleVmProfileCmd.java

@@ -102,7 +102,7 @@ public class CreateAutoScaleVmProfileCmd extends BaseAsyncCreateCmd {
             description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " +
                     "This binary data must be base64 encoded before adding it to the request. " +
                     "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
-                    "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." +
+                    "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
                     "You also need to change vm.userdata.max.length value",
             length = 1048576,
             since = "4.18.0")

api/src/main/java/org/apache/cloudstack/api/command/user/autoscale/UpdateAutoScaleVmProfileCmd.java

@@ -97,7 +97,7 @@ public class UpdateAutoScaleVmProfileCmd extends BaseAsyncCustomIdCmd {
             description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " +
                     "This binary data must be base64 encoded before adding it to the request. " +
                     "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
-                    "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." +
+                    "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
                     "You also need to change vm.userdata.max.length value",
             length = 1048576,
             since = "4.18.0")

api/src/main/java/org/apache/cloudstack/api/command/user/userdata/RegisterUserDataCmd.java

@@ -74,7 +74,14 @@ public class RegisterUserDataCmd extends BaseCmd {
     @Parameter(name = ApiConstants.PROJECT_ID, type = CommandType.UUID, entityType = ProjectResponse.class, description = "an optional project for the userdata")
     private Long projectId;
 
-    @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING, required = true, description = "Userdata content", length = 1048576)
+    @Parameter(name = ApiConstants.USER_DATA,
+            type = CommandType.STRING,
+            required = true,
+            description = "Base64 encoded userdata content. " +
+                    "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
+                    "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
+                    "You also need to change vm.userdata.max.length value",
+            length = 1048576)
     private String userData;
 
     @Parameter(name = ApiConstants.PARAMS, type = CommandType.STRING, description = "comma separated list of variables declared in userdata content")

api/src/main/java/org/apache/cloudstack/api/command/user/vm/DeployVMCmd.java

@@ -156,7 +156,11 @@ public class DeployVMCmd extends BaseAsyncCreateCustomIdCmd implements SecurityG
     private String hypervisor;
 
     @Parameter(name = ApiConstants.USER_DATA, type = CommandType.STRING,
-            description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. This binary data must be base64 encoded before adding it to the request. Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding.",
+            description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " +
+                    "This binary data must be base64 encoded before adding it to the request. " +
+                    "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
+                    "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
+                    "You also need to change vm.userdata.max.length value",
             length = 1048576)
     private String userData;
 

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ListVnfAppliancesCmd.java

@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.cloudstack.api.command.user.vm;
+
+import com.cloud.vm.VirtualMachine;
+import org.apache.cloudstack.acl.RoleType;
+import org.apache.cloudstack.api.APICommand;
+import org.apache.cloudstack.api.ResponseObject.ResponseView;
+import org.apache.cloudstack.api.command.user.UserCmd;
+
+import org.apache.cloudstack.api.response.UserVmResponse;
+
+@APICommand(name = "listVnfAppliances", description = "List VNF appliance owned by the account.",
+        responseObject = UserVmResponse.class,
+        responseView = ResponseView.Restricted, entityType = {VirtualMachine.class},
+        requestHasSensitiveInfo = false, responseHasSensitiveInfo = false,
+        authorized = {RoleType.Admin, RoleType.ResourceAdmin, RoleType.DomainAdmin, RoleType.User},
+        since = "4.19.1")
+public class ListVnfAppliancesCmd extends ListVMsCmd implements UserCmd {
+
+    @Override
+    public Boolean getVnf() {
+        return true;
+    }
+}

api/src/main/java/org/apache/cloudstack/api/command/user/vm/ResetVMUserDataCmd.java

@@ -62,7 +62,7 @@ public class ResetVMUserDataCmd extends BaseCmd implements UserCmd {
             description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " +
                     "This binary data must be base64 encoded before adding it to the request. " +
                     "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
-                    "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." +
+                    "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
                     "You also need to change vm.userdata.max.length value",
             length = 1048576)
     private String userData;

api/src/main/java/org/apache/cloudstack/api/command/user/vm/UpdateVMCmd.java

@@ -86,7 +86,7 @@ public class UpdateVMCmd extends BaseCustomIdCmd implements SecurityGroupAction,
                description = "an optional binary data that can be sent to the virtual machine upon a successful deployment. " +
                        "This binary data must be base64 encoded before adding it to the request. " +
                        "Using HTTP GET (via querystring), you can send up to 4KB of data after base64 encoding. " +
-                       "Using HTTP POST(via POST body), you can send up to 1MB of data after base64 encoding." +
+                       "Using HTTP POST (via POST body), you can send up to 1MB of data after base64 encoding. " +
                        "You also need to change vm.userdata.max.length value",
                length = 1048576,
                since = "4.16.0")

api/src/main/java/org/apache/cloudstack/api/response/AutoScaleVmProfileResponse.java

@@ -69,7 +69,7 @@ public class AutoScaleVmProfileResponse extends BaseResponse implements Controll
     private Map<String, String> counterParams;
 
     @SerializedName(ApiConstants.USER_DATA)
-    @Param(description = "Base 64 encoded VM user data")
+    @Param(description = "Base64 encoded VM user data")
     private String userData;
 
     @SerializedName(ApiConstants.USER_DATA_ID) @Param(description="the id of userdata used for the VM", since = "4.18.1")

api/src/main/java/org/apache/cloudstack/api/response/ImageStoreResponse.java

@@ -27,39 +27,39 @@
 
 @EntityReference(value = ImageStore.class)
 public class ImageStoreResponse extends BaseResponseWithAnnotations {
-    @SerializedName("id")
+    @SerializedName(ApiConstants.ID)
     @Param(description = "the ID of the image store")
     private String id;
 
-    @SerializedName("zoneid")
+    @SerializedName(ApiConstants.ZONE_ID)
     @Param(description = "the Zone ID of the image store")
     private String zoneId;
 
     @SerializedName(ApiConstants.ZONE_NAME)
     @Param(description = "the Zone name of the image store")
     private String zoneName;
 
-    @SerializedName("name")
+    @SerializedName(ApiConstants.NAME)
     @Param(description = "the name of the image store")
     private String name;
 
-    @SerializedName("url")
+    @SerializedName(ApiConstants.URL)
     @Param(description = "the url of the image store")
     private String url;
 
-    @SerializedName("protocol")
+    @SerializedName(ApiConstants.PROTOCOL)
     @Param(description = "the protocol of the image store")
     private String protocol;
 
     @SerializedName("providername")
     @Param(description = "the provider name of the image store")
     private String providerName;
 
-    @SerializedName("scope")
+    @SerializedName(ApiConstants.SCOPE)
     @Param(description = "the scope of the image store")
     private ScopeType scope;
 
-    @SerializedName("readonly")
+    @SerializedName(ApiConstants.READ_ONLY)
     @Param(description = "defines if store is read-only")
     private Boolean readonly;
 

api/src/main/java/org/apache/cloudstack/api/response/VMUserDataResponse.java

@@ -30,7 +30,7 @@ public class VMUserDataResponse extends BaseResponse {
     private String vmId;
 
     @SerializedName(ApiConstants.USER_DATA)
-    @Param(description = "Base 64 encoded VM user data")
+    @Param(description = "Base64 encoded VM user data")
     private String userData;
 
     public void setUserData(String userData) {

api/src/main/java/org/apache/cloudstack/userdata/UserDataManager.java

@@ -17,11 +17,16 @@
 package org.apache.cloudstack.userdata;
 
 import org.apache.cloudstack.api.BaseCmd;
+import org.apache.cloudstack.framework.config.ConfigKey;
 import org.apache.cloudstack.framework.config.Configurable;
 
 import com.cloud.utils.component.Manager;
 
 public interface UserDataManager extends Manager, Configurable {
+    String VM_USERDATA_MAX_LENGTH_STRING = "vm.userdata.max.length";
+    ConfigKey<Integer> VM_USERDATA_MAX_LENGTH = new ConfigKey<>("Advanced", Integer.class, VM_USERDATA_MAX_LENGTH_STRING, "32768",
+            "Max length of vm userdata after base64 encoding. Default is 32768 and maximum is 1048576", true);
+
     String concatenateUserData(String userdata1, String userdata2, String userdataProvider);
     String validateUserData(String userData, BaseCmd.HTTPMethod httpmethod);
 }

api/src/main/java/org/apache/cloudstack/vm/UnmanagedVMsManager.java

@@ -30,6 +30,15 @@ public interface UnmanagedVMsManager extends VmImportService, UnmanageVMService,
             "If set to true, do not remove VM nics (and its MAC addresses) when unmanaging a VM, leaving them allocated but not reserved. " +
                     "If set to false, nics are removed and MAC addresses can be reassigned", true, ConfigKey.Scope.Zone);
 
+    ConfigKey<Integer> RemoteKvmInstanceDisksCopyTimeout = new ConfigKey<>(Integer.class,
+            "remote.kvm.instance.disks.copy.timeout",
+            "Advanced",
+            "30",
+            "Timeout (in mins) to prepare and copy the disks of remote KVM instance while importing the instance from an external host",
+            true,
+            ConfigKey.Scope.Global,
+            null);
+
     static boolean isSupported(Hypervisor.HypervisorType hypervisorType) {
         return hypervisorType == VMware || hypervisorType == KVM;
     }

core/src/main/java/com/cloud/agent/api/CopyRemoteVolumeCommand.java

@@ -23,14 +23,11 @@
 
 @LogLevel(LogLevel.Log4jLevel.Trace)
 public class CopyRemoteVolumeCommand extends Command {
-
     String remoteIp;
     String username;
     String password;
     String srcFile;
-
     String tmpPath;
-
     StorageFilerTO storageFilerTO;
 
     public CopyRemoteVolumeCommand(String remoteIp, String username, String password) {