Merge branch '4.18'

LINBIT · Sep 7, 2023 · f6b2a58 · f6b2a58
2 parents 54a7a5d + f049f54
commit f6b2a58
Show file tree

Hide file tree

Showing 15 changed files with 169 additions and 58 deletions.
diff --git a/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java b/core/src/main/java/org/apache/cloudstack/direct/download/HttpsDirectTemplateDownloader.java
@@ -183,7 +183,8 @@ public Long getRemoteFileSize(String url, String format) {
                 SSLContext context = getSSLContext();
                 urlConnection.setSSLSocketFactory(context.getSocketFactory());
                 urlConnection.connect();
-                return QCOW2Utils.getVirtualSize(urlConnection.getInputStream());
+                boolean isCompressed = !url.endsWith("qcow2");
+                return QCOW2Utils.getVirtualSize(urlObj.openStream(), isCompressed);
             } catch (IOException e) {
                 throw new CloudRuntimeException(String.format("Cannot obtain qcow2 virtual size due to: %s", e.getMessage()), e);
             }

diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DatabaseAccessObject.java
@@ -18,6 +18,7 @@
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 
 import org.apache.log4j.Logger;
@@ -84,6 +85,33 @@ public boolean columnExists(Connection conn, String tableName, String columnName
         return columnExists;
     }
 
+    public String generateIndexName(String tableName, String columnName) {
+        return String.format("i_%s__%s", tableName, columnName);
+    }
+
+    public boolean indexExists(Connection conn, String tableName, String indexName) {
+        try (PreparedStatement pstmt = conn.prepareStatement(String.format("SHOW INDEXES FROM %s where Key_name = \"%s\"", tableName, indexName))) {
+            ResultSet result = pstmt.executeQuery();
+            if (result.next()) {
+                return true;
+            }
+        } catch (SQLException e) {
+            s_logger.debug(String.format("Index %s doesn't exist, ignoring exception:", indexName, e.getMessage()));
+        }
+        return false;
+    }
+
+    public void createIndex(Connection conn, String tableName, String columnName, String indexName) {
+        String stmt = String.format("CREATE INDEX %s on %s (%s)", indexName, tableName, columnName);
+        s_logger.debug("Statement: " + stmt);
+        try (PreparedStatement pstmt = conn.prepareStatement(stmt)) {
+            pstmt.execute();
+            s_logger.debug(String.format("Created index %s", indexName));
+        } catch (SQLException e) {
+            s_logger.warn(String.format("Unable to create index %s", indexName), e);
+        }
+    }
+
     protected static void closePreparedStatement(PreparedStatement pstmt, String errorMessage) {
         try {
             if (pstmt != null) {
@@ -93,5 +121,4 @@ protected static void closePreparedStatement(PreparedStatement pstmt, String err
             s_logger.warn(errorMessage, e);
         }
     }
-
 }
diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/DbUpgradeUtils.java
@@ -23,6 +23,14 @@ public class DbUpgradeUtils {
 
     private static DatabaseAccessObject dao = new DatabaseAccessObject();
 
+    public static void addIndexIfNeeded(Connection conn, String tableName, String columnName) {
+        String indexName = dao.generateIndexName(tableName, columnName);
+
+        if (!dao.indexExists(conn, tableName, indexName)) {
+            dao.createIndex(conn, tableName, columnName, indexName);
+        }
+    }
+
     public static void addForeignKey(Connection conn, String tableName, String tableColumn, String foreignTableName, String foreignColumnName) {
         dao.addForeignKey(conn, tableName, tableColumn, foreignTableName, foreignColumnName);
     }

diff --git a/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java b/engine/schema/src/main/java/com/cloud/upgrade/dao/Upgrade41800to41810.java
@@ -69,6 +69,7 @@ public void performDataMigration(Connection conn) {
         copyGuestOsMappingsToVMware80u1();
         addForeignKeyToAutoscaleVmprofiles(conn);
         mergeDuplicateGuestOSes();
+        addIndexes(conn);
     }
 
     private void mergeDuplicateGuestOSes() {
@@ -242,4 +243,8 @@ private void fixForeignKeyNames(Connection conn) {
     private void addForeignKeyToAutoscaleVmprofiles(Connection conn) {
         DbUpgradeUtils.addForeignKey(conn, "autoscale_vmprofiles", "user_data_id", "user_data", "id");
     }
+
+    private void addIndexes(Connection conn) {
+        DbUpgradeUtils.addIndexIfNeeded(conn, "cluster_details", "name");
+    }
 }
diff --git a/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java b/engine/schema/src/test/java/com/cloud/upgrade/dao/DatabaseAccessObjectTest.java
@@ -16,6 +16,7 @@
 // under the License.
 package com.cloud.upgrade.dao;
 
+import static org.mockito.Matchers.startsWith;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.anyString;
 import static org.mockito.Matchers.contains;
@@ -27,9 +28,11 @@
 
 import java.sql.Connection;
 import java.sql.PreparedStatement;
+import java.sql.ResultSet;
 import java.sql.SQLException;
 
 import org.apache.log4j.Logger;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -49,6 +52,9 @@ public class DatabaseAccessObjectTest {
     @Mock
     private Logger loggerMock;
 
+    @Mock
+    private ResultSet resultSetMock;
+
     private final DatabaseAccessObject dao = new DatabaseAccessObject();
 
     @Before
@@ -83,6 +89,61 @@ public void testDropKeyWhenConnectionIsNull() throws Exception {
         dao.dropKey(conn, tableName, key, isForeignKey);
     }
 
+    @Test
+    public void generateIndexNameTest() {
+        String indexName = dao.generateIndexName("mytable","mycolumn");
+        Assert.assertEquals( "i_mytable__mycolumn", indexName);
+    }
+
+    @Test
+    public void indexExistsFalseTest() throws Exception {
+        when(resultSetMock.next()).thenReturn(false);
+        when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock);
+        when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock);
+
+        Connection conn = connectionMock;
+        String tableName = "mytable";
+        String indexName = "myindex";
+
+        Assert.assertFalse(dao.indexExists(conn, tableName, indexName));
+        verify(connectionMock, times(1)).prepareStatement(anyString());
+        verify(preparedStatementMock, times(1)).executeQuery();
+        verify(preparedStatementMock, times(1)).close();
+    }
+
+    @Test
+    public void indexExistsTrueTest() throws Exception {
+        when(resultSetMock.next()).thenReturn(true);
+        when(connectionMock.prepareStatement(startsWith("SHOW INDEXES FROM"))).thenReturn(preparedStatementMock);
+        when(preparedStatementMock.executeQuery()).thenReturn(resultSetMock);
+
+        Connection conn = connectionMock;
+        String tableName = "mytable";
+        String indexName = "myindex";
+
+        Assert.assertTrue(dao.indexExists(conn, tableName, indexName));
+        verify(connectionMock, times(1)).prepareStatement(anyString());
+        verify(preparedStatementMock, times(1)).executeQuery();
+        verify(preparedStatementMock, times(1)).close();
+    }
+
+    @Test
+    public void createIndexTest() throws Exception {
+        when(connectionMock.prepareStatement(startsWith("CREATE INDEX"))).thenReturn(preparedStatementMock);
+        when(preparedStatementMock.execute()).thenReturn(true);
+
+        Connection conn = connectionMock;
+        String tableName = "mytable";
+        String columnName = "mycolumn";
+        String indexName = "myindex";
+
+        dao.createIndex(conn, tableName, columnName, indexName);
+        verify(connectionMock, times(1)).prepareStatement(anyString());
+        verify(preparedStatementMock, times(1)).execute();
+        verify(preparedStatementMock, times(1)).close();
+        verify(loggerMock, times(1)).debug("Created index myindex");
+    }
+
     @Test
     public void testDropKeyWhenTableNameIsNull() throws Exception {
         SQLException sqlException = new SQLException();

diff --git a/...ns/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java b/...ns/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/LibvirtConnection.java
@@ -21,7 +21,6 @@
 
 import org.apache.log4j.Logger;
 import org.libvirt.Connect;
-import org.libvirt.Library;
 import org.libvirt.LibvirtException;
 
 import com.cloud.hypervisor.Hypervisor;
@@ -45,7 +44,6 @@ static public Connect getConnection(String hypervisorURI) throws LibvirtExceptio
         if (conn == null) {
             s_logger.info("No existing libvirtd connection found. Opening a new one");
             conn = new Connect(hypervisorURI, false);
-            Library.initEventLoop();
             s_logger.debug("Successfully connected to libvirt at: " + hypervisorURI);
             s_connections.put(hypervisorURI, conn);
         } else {

diff --git a/...s/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java b/...s/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtCheckUrlCommand.java
@@ -40,6 +40,11 @@ public CheckUrlAnswer execute(CheckUrlCommand cmd, LibvirtComputingResource serv
         boolean checkResult = DirectDownloadHelper.checkUrlExistence(url);
         if (checkResult) {
             remoteSize = DirectDownloadHelper.getFileSize(url, cmd.getFormat());
+            if (remoteSize == null || remoteSize < 0) {
+                s_logger.error(String.format("Couldn't properly retrieve the remote size of the template on " +
+                        "url %s, obtained size = %s", url, remoteSize));
+                return new CheckUrlAnswer(false, remoteSize);
+            }
         }
         return new CheckUrlAnswer(checkResult, remoteSize);
     }

diff --git a/...ns/network-elements/vxlan/src/main/java/com/cloud/network/guru/VxlanGuestNetworkGuru.java b/...ns/network-elements/vxlan/src/main/java/com/cloud/network/guru/VxlanGuestNetworkGuru.java
@@ -69,19 +69,16 @@ protected boolean canHandle(NetworkOffering offering, final NetworkType networkT
 
     @Override
     public Network design(NetworkOffering offering, DeploymentPlan plan, Network userSpecified, Account owner) {
-
         NetworkVO network = (NetworkVO)super.design(offering, plan, userSpecified, owner);
         if (network == null) {
             return null;
         }
-
         if (offering.getGuestType() == GuestType.L2 && network.getBroadcastUri() != null) {
             String vxlan = BroadcastDomainType.getValue(network.getBroadcastUri());
             network.setBroadcastUri(BroadcastDomainType.Vxlan.toUri(vxlan));
         }
         network.setBroadcastDomainType(BroadcastDomainType.Vxlan);
-
-        return network;
+        return updateNetworkDesignForIPv6IfNeeded(network, userSpecified);
     }
 
     @Override

diff --git a/...rc/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java b/...rc/main/java/org/apache/cloudstack/storage/datastore/client/ScaleIOGatewayClientImpl.java
@@ -738,11 +738,20 @@ public boolean deleteVolume(final String volumeId) {
         try {
             unmapVolumeFromAllSdcs(volumeId);
         } catch (Exception ignored) {}
-        Boolean removeVolumeStatus = post(
-                "/instances/Volume::" + volumeId + "/action/removeVolume",
-                "{\"removeMode\":\"ONLY_ME\"}", Boolean.class);
-        if (removeVolumeStatus != null) {
-            return removeVolumeStatus;
+
+        try {
+            Boolean removeVolumeStatus = post(
+                    "/instances/Volume::" + volumeId + "/action/removeVolume",
+                    "{\"removeMode\":\"ONLY_ME\"}", Boolean.class);
+            if (removeVolumeStatus != null) {
+                return removeVolumeStatus;
+            }
+        } catch (Exception ex) {
+            if (ex instanceof ServerApiException && ex.getMessage().contains("Could not find the volume")) {
+                LOG.warn(String.format("API says deleting volume %s does not exist, handling gracefully", volumeId));
+                return true;
+            }
+            throw ex;
         }
         return false;
     }

diff --git a/server/src/main/java/com/cloud/network/guru/ExternalGuestNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/ExternalGuestNetworkGuru.java
@@ -36,7 +36,6 @@
 import com.cloud.event.EventVO;
 import com.cloud.exception.InsufficientAddressCapacityException;
 import com.cloud.exception.InsufficientVirtualNetworkCapacityException;
-import com.cloud.exception.InvalidParameterValueException;
 import com.cloud.network.IpAddressManager;
 import com.cloud.network.Network;
 import com.cloud.network.Network.GuestType;
@@ -124,22 +123,7 @@ public Network design(NetworkOffering offering, DeploymentPlan plan, Network use
             /* In order to revert userSpecified network setup */
             config.setState(State.Allocated);
         }
-        if (userSpecified == null) {
-            return config;
-        }
-        if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) ||
-                    (userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) {
-            throw new InvalidParameterValueException("ip6gateway and ip6cidr must be specified together.");
-        }
-        if (userSpecified.getIp6Cidr() != null) {
-            config.setIp6Cidr(userSpecified.getIp6Cidr());
-            config.setIp6Gateway(userSpecified.getIp6Gateway());
-        }
-        if (userSpecified.getRouterIpv6() != null) {
-            config.setRouterIpv6(userSpecified.getRouterIpv6());
-        }
-
-        return config;
+        return updateNetworkDesignForIPv6IfNeeded(config, userSpecified);
     }
 
     @Override

diff --git a/server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java b/server/src/main/java/com/cloud/network/guru/GuestNetworkGuru.java
@@ -545,4 +545,22 @@ public String getConfigComponentName() {
     public ConfigKey<?>[] getConfigKeys() {
         return new ConfigKey<?>[]{UseSystemGuestVlans};
     }
+
+    public Network updateNetworkDesignForIPv6IfNeeded(NetworkVO network, Network userSpecified) {
+        if (userSpecified == null) {
+            return network;
+        }
+        if ((userSpecified.getIp6Cidr() == null && userSpecified.getIp6Gateway() != null) ||
+                (userSpecified.getIp6Cidr() != null && userSpecified.getIp6Gateway() == null)) {
+            throw new InvalidParameterValueException("ip6gateway and ip6cidr must be specified together.");
+        }
+        if (userSpecified.getIp6Cidr() != null) {
+            network.setIp6Cidr(userSpecified.getIp6Cidr());
+            network.setIp6Gateway(userSpecified.getIp6Gateway());
+        }
+        if (userSpecified.getRouterIpv6() != null) {
+            network.setRouterIpv6(userSpecified.getRouterIpv6());
+        }
+        return network;
+    }
 }
diff --git a/...rage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java b/...rage/server/src/main/java/org/apache/cloudstack/storage/template/DownloadManagerImpl.java
@@ -347,7 +347,7 @@ private String postRemoteDownload(String jobId) {
             // and as such can be easily read.
 
             try (InputStream inputStream = td.getS3ObjectInputStream();) {
-                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream));
+                dnld.setTemplatesize(QCOW2Utils.getVirtualSize(inputStream, false));
             }
             catch (IOException e) {
                 result = "Couldn't read QCOW2 virtual size. Error: " + e.getMessage();

diff --git a/ui/src/views/AutogenView.vue b/ui/src/views/AutogenView.vue
@@ -968,10 +968,6 @@ export default {
           console.log('DEBUG - Discarding API response as its `id` does not match the uuid on the browser path')
           return
         }
-        if (this.dataView && apiItemCount > 1) {
-          console.log('DEBUG - Discarding API response as got more than one item in data view', this.$route.params, this.items)
-          return
-        }
 
         this.items = json[responseName][objectName]
         if (!this.items || this.items.length === 0) {