Linstor: Allow snapshot backup also to work on non hyperconverged setups

On no access to the storage nodes, we now create a temporary resource
from the snapshot and copy that data into the secondary storage.
Revert works the same, just that we now also look additionally
for any Linstor agent node.

Also enables now backup snapshot by default.

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/driver/LinstorPrimaryDataStoreDriverImpl.java

@@ -527,6 +527,16 @@ private String cloneResource(long csCloneId, VolumeInfo volumeInfo, StoragePoolV
         }
     }
 
+    private ResourceDefinitionCreate createResourceDefinitionCreate(String rscName, String rscGrpName)
+            throws ApiException {
+        ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate();
+        ResourceDefinition rd = new ResourceDefinition();
+        rd.setName(rscName);
+        rd.setResourceGroupName(rscGrpName);
+        rdCreate.setResourceDefinition(rd);
+        return rdCreate;
+    }
+
     private String createResourceFromSnapshot(long csSnapshotId, String rscName, StoragePoolVO storagePoolVO) {
         final String rscGrp = getRscGrp(storagePoolVO);
         final DevelopersApi linstorApi = LinstorUtil.getLinstorAPI(storagePoolVO.getHostAddress());
@@ -539,11 +549,7 @@ private String createResourceFromSnapshot(long csSnapshotId, String rscName, Sto
         try
         {
             s_logger.debug("Create new resource definition: " + rscName);
-            ResourceDefinitionCreate rdCreate = new ResourceDefinitionCreate();
-            ResourceDefinition rd = new ResourceDefinition();
-            rd.setName(rscName);
-            rd.setResourceGroupName(rscGrp);
-            rdCreate.setResourceDefinition(rd);
+            ResourceDefinitionCreate rdCreate = createResourceDefinitionCreate(rscName, rscGrp);
             ApiCallRcList answers = linstorApi.resourceDefinitionCreate(rdCreate);
             checkLinstorAnswersThrow(answers);
 
@@ -712,6 +718,10 @@ private String revertSnapshotFromImageStore(
             VirtualMachineManager.ExecuteInSequence.value());
 
         Optional<RemoteHostEndPoint> optEP = getDiskfullEP(linstorApi, rscName);
+        if (optEP.isEmpty()) {
+            optEP = getLinstorEP(linstorApi, rscName);
+        }
+
         if (optEP.isPresent()) {
             Answer answer = optEP.get().sendMessage(cmd);
             if (!answer.getResult()) {
@@ -840,6 +850,14 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
         callback.complete(res);
     }
 
+    /**
+     * Tries to get a Linstor cloudstack end point, that is at least diskless.
+     *
+     * @param api Linstor java api object
+     * @param rscName resource name to make available on node
+     * @return Optional RemoteHostEndPoint if one could get found.
+     * @throws ApiException
+     */
     private Optional<RemoteHostEndPoint> getLinstorEP(DevelopersApi api, String rscName) throws ApiException {
         List<String> linstorNodeNames = LinstorUtil.getLinstorNodeNames(api);
         Collections.shuffle(linstorNodeNames);  // do not always pick the first linstor node
@@ -892,6 +910,25 @@ private Optional<RemoteHostEndPoint> getDiskfullEP(DevelopersApi api, String rsc
         return Optional.empty();
     }
 
+    private String restoreResourceFromSnapshot(
+            DevelopersApi api,
+            StoragePoolVO storagePoolVO,
+            String rscName,
+            String snapshotName,
+            String restoredName) throws ApiException {
+        final String rscGrp = getRscGrp(storagePoolVO);
+        ResourceDefinitionCreate rdc = createResourceDefinitionCreate(restoredName, rscGrp);
+        api.resourceDefinitionCreate(rdc);
+
+        SnapshotRestore sr = new SnapshotRestore();
+        sr.toResource(restoredName);
+        api.resourceSnapshotsRestoreVolumeDefinition(rscName, snapshotName, sr);
+
+        api.resourceSnapshotRestore(rscName, snapshotName, sr);
+
+        return getDeviceName(api, restoredName);
+    }
+
     private Answer copyTemplate(DataObject srcData, DataObject dstData) {
         TemplateInfo tInfo = (TemplateInfo) dstData;
         final StoragePoolVO pool = _storagePoolDao.findById(dstData.getDataStore().getId());
@@ -956,13 +993,30 @@ protected Answer copySnapshot(DataObject srcData, DataObject destData) {
                 VirtualMachineManager.ExecuteInSequence.value());
             cmd.setOptions(options);
 
-            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(
-                api, LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid());
+            String rscName = LinstorUtil.RSC_PREFIX + snapshotInfo.getBaseVolume().getUuid();
+            Optional<RemoteHostEndPoint> optEP = getDiskfullEP(api, rscName);
             Answer answer;
             if (optEP.isPresent()) {
                 answer = optEP.get().sendMessage(cmd);
             } else {
-                answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+                s_logger.debug("No diskfull endpoint found to copy image, creating diskless endpoint");
+                // create a temporary resource from the snapshot to backup, so we can copy the data on a diskless agent
+                String restoreName = rscName + "-rst";
+                String snapshotName = LinstorUtil.RSC_PREFIX + snapshotInfo.getUuid();
+                String devName = restoreResourceFromSnapshot(api, pool, rscName, snapshotName, restoreName);
+
+                Optional<RemoteHostEndPoint> optEPAny = getLinstorEP(api, restoreName);
+                if (optEPAny.isPresent()) {
+                    // patch the src device path to the temporary linstor resource
+                    SnapshotObjectTO soTO = (SnapshotObjectTO)srcData.getTO();
+                    soTO.setPath(devName);
+                    cmd.setSrcTO(soTO);
+                    answer = optEPAny.get().sendMessage(cmd);
+                } else{
+                    answer = new Answer(cmd, false, "Unable to get matching Linstor endpoint.");
+                }
+                // delete the temporary resource, noop if already gone
+                api.resourceDefinitionDelete(restoreName);
             }
             return answer;
         } catch (Exception e) {

plugins/storage/volume/linstor/src/main/java/org/apache/cloudstack/storage/datastore/util/LinstorConfigurationManager.java

@@ -21,8 +21,8 @@
 
 public class LinstorConfigurationManager implements Configurable
 {
-    public static final ConfigKey<Boolean> BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "false",
-        "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot), only works on hyperconverged setups.", true, ConfigKey.Scope.Global, null);
+    public static final ConfigKey<Boolean> BackupSnapshots = new ConfigKey<>(Boolean.class, "lin.backup.snapshots", "Advanced", "true",
+        "Backup Linstor primary storage snapshots to secondary storage (deleting ps snapshot)", true, ConfigKey.Scope.Global, null);
 
     public static final ConfigKey<?>[] CONFIG_KEYS = new ConfigKey<?>[] { BackupSnapshots };