Apple FR76: Implicit host tags (apache#427)

* Merge two HostTagVO and HostTagDaoImpl

* Apple FR76: dynamic host tags

* Revert "Apple FR76: dynamic host tags"

This reverts commit 01b93a8.

* Apple FR76: Implicit host tags

* Apple FR76: address Abhishek's comments

* Apple FR76: move updateImplicitTags

* Apple FR76: add since to other two responses

* Update 8929: add unit test in LibvirtComputingResourceTest

* Update variable names

* Update FR76: add explicithosttags in response

* Update FR76 UI: Update explicit host tags

* Update 8929: remove host tags and change labels on UI

* Update: ui polish for host tags

* fix since in responses

* Update 8929: fix UI error if no host tags

agent/conf/agent.properties

@@ -415,3 +415,6 @@ iscsi.session.cleanup.enabled=false
 # If set to "true", the agent will register for libvirt domain events, allowing for immediate updates on crashed or
 # unexpectedly stopped. Experimental, requires agent restart.
 # libvirt.events.enabled=false
+
+# Implicit host tags managed by agent.properties
+# host.tags=

agent/src/main/java/com/cloud/agent/properties/AgentProperties.java

@@ -755,6 +755,13 @@ public Property<Integer> getWorkers() {
      */
     public static final Property<String> CONTROL_CIDR = new Property<>("control.cidr", "169.254.0.0/16");
 
+    /**
+     * Implicit host tags
+     * Data type: String.<br>
+     * Default value: <code>null</code>
+     */
+    public static final Property<String> HOST_TAGS = new Property<>("host.tags", null, String.class);
+
     public static class Property <T>{
         private String name;
         private T defaultValue;

api/src/main/java/org/apache/cloudstack/api/ApiConstants.java

@@ -252,6 +252,7 @@ public class ApiConstants {
     public static final String IS_EDGE = "isedge";
     public static final String IS_EXTRACTABLE = "isextractable";
     public static final String IS_FEATURED = "isfeatured";
+    public static final String IS_IMPLICIT = "isimplicit";
     public static final String IS_PORTABLE = "isportable";
     public static final String IS_PUBLIC = "ispublic";
     public static final String IS_PERSISTENT = "ispersistent";

api/src/main/java/org/apache/cloudstack/api/response/HostForMigrationResponse.java

@@ -208,6 +208,14 @@ public class HostForMigrationResponse extends BaseResponse {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.18.1")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.18.1")
+    private String implicitHostTags;
+
     @SerializedName("hasenoughcapacity")
     @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
     private Boolean hasEnoughCapacity;
@@ -414,6 +422,14 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostResponse.java

@@ -221,6 +221,14 @@ public class HostResponse extends BaseResponseWithAnnotations {
     @Param(description = "comma-separated list of tags for the host")
     private String hostTags;
 
+    @SerializedName("explicithosttags")
+    @Param(description = "comma-separated list of explicit host tags for the host", since = "4.18.1")
+    private String explicitHostTags;
+
+    @SerializedName("implicithosttags")
+    @Param(description = "comma-separated list of implicit host tags for the host", since = "4.18.1")
+    private String implicitHostTags;
+
     @SerializedName("hasenoughcapacity")
     @Param(description = "true if this host has enough CPU and RAM capacity to migrate a VM to it, false otherwise")
     private Boolean hasEnoughCapacity;
@@ -454,6 +462,22 @@ public void setHostTags(String hostTags) {
         this.hostTags = hostTags;
     }
 
+    public String getExplicitHostTags() {
+        return explicitHostTags;
+    }
+
+    public void setExplicitHostTags(String explicitHostTags) {
+        this.explicitHostTags = explicitHostTags;
+    }
+
+    public String getImplicitHostTags() {
+        return implicitHostTags;
+    }
+
+    public void setImplicitHostTags(String implicitHostTags) {
+        this.implicitHostTags = implicitHostTags;
+    }
+
     public void setHasEnoughCapacity(Boolean hasEnoughCapacity) {
         this.hasEnoughCapacity = hasEnoughCapacity;
     }

api/src/main/java/org/apache/cloudstack/api/response/HostTagResponse.java

@@ -19,6 +19,7 @@
 import com.google.gson.annotations.SerializedName;
 import com.cloud.serializer.Param;
 
+import org.apache.cloudstack.api.ApiConstants;
 import org.apache.cloudstack.api.BaseResponse;
 
 public class HostTagResponse extends BaseResponse {
@@ -34,6 +35,10 @@ public class HostTagResponse extends BaseResponse {
     @Param(description = "the name of the host tag")
     private String name;
 
+    @SerializedName(ApiConstants.IS_IMPLICIT)
+    @Param(description = "true if the host tag is implicit", since = "4.18.1")
+    private boolean isImplicit;
+
     public String getId() {
         return id;
     }
@@ -57,4 +62,12 @@ public String getName() {
     public void setName(String name) {
         this.name = name;
     }
+
+    public boolean isImplicit() {
+        return isImplicit;
+    }
+
+    public void setImplicit(boolean implicit) {
+        isImplicit = implicit;
+    }
 }

core/src/main/java/com/cloud/agent/api/StartupRoutingCommand.java

@@ -174,6 +174,10 @@ public void setHostTags(String hostTag) {
         this.hostTags.add(hostTag);
     }
 
+    public void setHostTags(List<String> hostTags) {
+        this.hostTags = hostTags;
+    }
+
     public  HashMap<String, HashMap<String, VgpuTypesInfo>> getGpuGroupDetails() {
         return groupDetails;
     }

engine/schema/src/main/java/com/cloud/host/HostTagVO.java

@@ -39,6 +39,9 @@ public class HostTagVO implements InternalIdentity {
     @Column(name = "tag")
     private String tag;
 
+    @Column(name = "is_implicit")
+    private boolean isImplicit = false;
+
     protected HostTagVO() {
     }
 
@@ -59,6 +62,14 @@ public void setTag(String tag) {
         this.tag = tag;
     }
 
+    public void setIsImplicit(boolean isImplicit) {
+        this.isImplicit = isImplicit;
+    }
+
+    public boolean getIsImplicit() {
+        return isImplicit;
+    }
+
     @Override
     public long getId() {
         return id;

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDao.java

@@ -21,6 +21,8 @@
 import com.cloud.host.HostTagVO;
 import com.cloud.utils.db.GenericDao;
 
+import org.apache.cloudstack.api.response.HostTagResponse;
+
 public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
 
     void persist(long hostId, List<String> hostTags);
@@ -31,4 +33,11 @@ public interface HostTagsDao extends GenericDao<HostTagVO, Long> {
 
     void deleteTags(long hostId);
 
+    boolean updateImplicitTags(long hostId, List<String> hostTags);
+
+    List<HostTagVO> getExplicitHostTags(long hostId);
+
+    HostTagResponse newHostTagResponse(HostTagVO hostTag);
+
+    List<HostTagVO> searchByIds(Long... hostTagIds);
 }

engine/schema/src/main/java/com/cloud/host/dao/HostTagsDaoImpl.java

@@ -19,7 +19,9 @@
 import java.util.ArrayList;
 import java.util.List;
 
-
+import org.apache.cloudstack.api.response.HostTagResponse;
+import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
+import org.apache.commons.lang3.StringUtils;
 import org.springframework.stereotype.Component;
 
 import com.cloud.host.HostTagVO;
@@ -30,21 +32,43 @@
 import com.cloud.utils.db.TransactionLegacy;
 import com.cloud.utils.db.SearchCriteria.Func;
 
+import javax.inject.Inject;
+
 @Component
 public class HostTagsDaoImpl extends GenericDaoBase<HostTagVO, Long> implements HostTagsDao {
     protected final SearchBuilder<HostTagVO> HostSearch;
     protected final GenericSearchBuilder<HostTagVO, String> DistinctImplictTagsSearch;
+    private final SearchBuilder<HostTagVO> stSearch;
+    private final SearchBuilder<HostTagVO> tagIdsearch;
+    private final SearchBuilder<HostTagVO> ImplicitTagsSearch;
+
+    @Inject
+    private ConfigurationDao _configDao;
 
     public HostTagsDaoImpl() {
         HostSearch = createSearchBuilder();
         HostSearch.and("hostId", HostSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        HostSearch.and("isImplicit", HostSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
         HostSearch.done();
 
         DistinctImplictTagsSearch = createSearchBuilder(String.class);
         DistinctImplictTagsSearch.select(null, Func.DISTINCT, DistinctImplictTagsSearch.entity().getTag());
         DistinctImplictTagsSearch.and("hostIds", DistinctImplictTagsSearch.entity().getHostId(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.and("implicitTags", DistinctImplictTagsSearch.entity().getTag(), SearchCriteria.Op.IN);
         DistinctImplictTagsSearch.done();
+
+        stSearch = createSearchBuilder();
+        stSearch.and("idIN", stSearch.entity().getId(), SearchCriteria.Op.IN);
+        stSearch.done();
+
+        tagIdsearch = createSearchBuilder();
+        tagIdsearch.and("id", tagIdsearch.entity().getId(), SearchCriteria.Op.EQ);
+        tagIdsearch.done();
+
+        ImplicitTagsSearch = createSearchBuilder();
+        ImplicitTagsSearch.and("hostId", ImplicitTagsSearch.entity().getHostId(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.and("isImplicit", ImplicitTagsSearch.entity().getIsImplicit(), SearchCriteria.Op.EQ);
+        ImplicitTagsSearch.done();
     }
 
     @Override
@@ -79,13 +103,44 @@ public void deleteTags(long hostId) {
         txn.commit();
     }
 
+    @Override
+    public boolean updateImplicitTags(long hostId, List<String> hostTags) {
+        TransactionLegacy txn = TransactionLegacy.currentTxn();
+        txn.start();
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", true);
+        boolean expunged = expunge(sc) > 0;
+        boolean persisted = false;
+        for (String tag : hostTags) {
+            if (StringUtils.isNotBlank(tag)) {
+                HostTagVO vo = new HostTagVO(hostId, tag.trim());
+                vo.setIsImplicit(true);
+                persist(vo);
+                persisted = true;
+            }
+        }
+        txn.commit();
+        return expunged || persisted;
+    }
+
+    @Override
+    public List<HostTagVO> getExplicitHostTags(long hostId) {
+        SearchCriteria<HostTagVO> sc = ImplicitTagsSearch.create();
+        sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
+
+        return search(sc, null);
+    }
+
     @Override
     public void persist(long hostId, List<String> hostTags) {
         TransactionLegacy txn = TransactionLegacy.currentTxn();
 
         txn.start();
         SearchCriteria<HostTagVO> sc = HostSearch.create();
         sc.setParameters("hostId", hostId);
+        sc.setParameters("isImplicit", false);
         expunge(sc);
 
         for (String tag : hostTags) {
@@ -97,4 +152,71 @@ public void persist(long hostId, List<String> hostTags) {
         }
         txn.commit();
     }
+
+    @Override
+    public HostTagResponse newHostTagResponse(HostTagVO tag) {
+        HostTagResponse tagResponse = new HostTagResponse();
+
+        tagResponse.setName(tag.getTag());
+        tagResponse.setHostId(tag.getHostId());
+
+        tagResponse.setObjectName("hosttag");
+
+        return tagResponse;
+    }
+
+    @Override
+    public List<HostTagVO> searchByIds(Long... tagIds) {
+        String batchCfg = _configDao.getValue("detail.batch.query.size");
+
+        final int detailsBatchSize = batchCfg != null ? Integer.parseInt(batchCfg) : 2000;
+
+        // query details by batches
+        List<HostTagVO> tagList = new ArrayList<>();
+        int curr_index = 0;
+
+        if (tagIds.length > detailsBatchSize) {
+            while ((curr_index + detailsBatchSize) <= tagIds.length) {
+                Long[] ids = new Long[detailsBatchSize];
+
+                for (int k = 0, j = curr_index; j < curr_index + detailsBatchSize; j++, k++) {
+                    ids[k] = tagIds[j];
+                }
+
+                SearchCriteria<HostTagVO> sc = stSearch.create();
+
+                sc.setParameters("idIN", (Object[])ids);
+
+                List<HostTagVO> vms = searchIncludingRemoved(sc, null, null, false);
+
+                if (vms != null) {
+                    tagList.addAll(vms);
+                }
+
+                curr_index += detailsBatchSize;
+            }
+        }
+
+        if (curr_index < tagIds.length) {
+            int batch_size = (tagIds.length - curr_index);
+            // set the ids value
+            Long[] ids = new Long[batch_size];
+
+            for (int k = 0, j = curr_index; j < curr_index + batch_size; j++, k++) {
+                ids[k] = tagIds[j];
+            }
+
+            SearchCriteria<HostTagVO> sc = stSearch.create();
+
+            sc.setParameters("idIN", (Object[])ids);
+
+            List<HostTagVO> tags = searchIncludingRemoved(sc, null, null, false);
+
+            if (tags != null) {
+                tagList.addAll(tags);
+            }
+        }
+
+        return tagList;
+    }
 }

engine/schema/src/main/resources/META-INF/cloudstack/core/spring-engine-schema-core-daos-context.xml

@@ -186,7 +186,6 @@
   <bean id="storageNetworkIpAddressDaoImpl" class="com.cloud.dc.dao.StorageNetworkIpAddressDaoImpl" />
   <bean id="storageNetworkIpRangeDaoImpl" class="com.cloud.dc.dao.StorageNetworkIpRangeDaoImpl" />
   <bean id="storagePoolJoinDaoImpl" class="com.cloud.api.query.dao.StoragePoolJoinDaoImpl" />
-  <bean id="hostTagDaoImpl" class="com.cloud.api.query.dao.HostTagDaoImpl" />
   <bean id="storagePoolWorkDaoImpl" class="com.cloud.storage.dao.StoragePoolWorkDaoImpl" />
   <bean id="uploadDaoImpl" class="com.cloud.storage.dao.UploadDaoImpl" />
   <bean id="usageDaoImpl" class="com.cloud.usage.dao.UsageDaoImpl" />