From a9da77d247f4bf874d38efe6c856cd342a3c1e9c Mon Sep 17 00:00:00 2001 From: Camelia Dumitru Date: Fri, 27 Sep 2024 15:32:14 +0100 Subject: [PATCH] Added the code to store deleted items in panoply dw_deleted_items table The items will be processed by the script that runs daily and cleaned up after the item is deleted from the corresponding view --- .../impl/OrgAffiliationRelationDaoImpl.java | 66 ++++++++++++++++++- .../resources/orcid-persistence-context.xml | 16 +++++ orcid-utils/pom.xml | 20 ++++++ .../utils/panoply/PanoplyDeletedItem.java | 55 ++++++++++++++++ .../utils/panoply/PanoplyRedshiftClient.java | 36 ++++++++++ properties/development.properties | 12 +++- 6 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index 51185a67132..de1c3a5bf1f 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,19 +3,32 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; +import org.orcid.utils.panoply.PanoplyDeletedItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - + + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); + + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0 ? true : false; + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setItemId(orgAffiliationRelationId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + return true; + } + return false; + } + + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { + //Store the deleted item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyDeletedItem(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store deleted affiliation in panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); } /** @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setClientSourceId(clientSourceId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } } @Override @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setOrcid(orcid); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } + } @Override diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index d57dde50df1..7857fb72b54 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,6 +457,22 @@ + + + + + + + + + + + + + + + + diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index d7b361ba22d..6ee5bc45646 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-context + + org.springframework + spring-jdbc + org.apache.commons commons-lang3 @@ -113,6 +117,22 @@ jsoup 1.15.4 + + + + com.amazon.redshift + redshift-jdbc42 + 2.1.0.30 + + + + + + com.zaxxer + HikariCP + 5.0.1 + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java new file mode 100644 index 00000000000..378b0185227 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -0,0 +1,55 @@ +package org.orcid.utils.panoply; + +public class PanoplyDeletedItem { + private Long id; + private String dwTable; + private Long itemId; + private String clientSourceId; + private String orcid; + + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; + public final String DW_WORK = "dw_work"; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getDwTable() { + return dwTable; + } + public void setDwTable(String dwTable) { + this.dwTable = dwTable; + } + public Long getItemId() { + return itemId; + } + public void setItemId(Long itemId) { + this.itemId = itemId; + } + public String getClientSourceId() { + return clientSourceId; + } + public void setClientSourceId(String clientSourceId) { + this.clientSourceId = clientSourceId; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + + @Override + public String toString() { + return "PanoplyDeletedItem{" + + "id=" + id + + ", dwTable='" + dwTable + '\'' + + ", itemId='" + itemId + '\'' + + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + + '}'; + } + +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java new file mode 100644 index 00000000000..3499593eb27 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -0,0 +1,36 @@ +package org.orcid.utils.panoply; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + + +import java.util.Date; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.orcid.utils.alerting.SlackManager; + +@Repository +public class PanoplyRedshiftClient { + + private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); + + @Autowired + @Qualifier("panoplyJdbcTemplate") + private JdbcTemplate panoplyJdbcTemplate; + + + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); + } + +} diff --git a/properties/development.properties b/properties/development.properties index 34eae804933..9186b22ceb8 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -#org.orcid.persistence.liquibase.enabled=false +org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx