From 6a075350a864a47ee12a1f540f246644ed1e9957 Mon Sep 17 00:00:00 2001 From: Angel Montenegro Date: Fri, 11 Oct 2024 12:08:54 -0600 Subject: [PATCH] Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table"" (#7100) * Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table"" * Fixed the failures in test suite --------- Co-authored-by: Camelia Dumitru --- .../impl/OrgAffiliationRelationDaoImpl.java | 66 ++++++++++++++++++- .../resources/orcid-persistence-context.xml | 21 +++++- .../properties/test-api-common.properties | 14 +++- .../properties/test-api-internal.properties | 14 +++- .../resources/properties/test-core.properties | 14 +++- .../resources/properties/test-db.properties | 12 ++++ .../properties/test-scheduler.properties | 12 ++++ .../resources/properties/test-ui.properties | 14 +++- orcid-utils/pom.xml | 20 ++++++ .../utils/panoply/PanoplyDeletedItem.java | 55 ++++++++++++++++ .../utils/panoply/PanoplyRedshiftClient.java | 36 ++++++++++ properties/development.properties | 12 +++- 12 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index 51185a67132..de1c3a5bf1f 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,19 +3,32 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; +import org.orcid.utils.panoply.PanoplyDeletedItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - + + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); + + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0 ? true : false; + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setItemId(orgAffiliationRelationId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + return true; + } + return false; + } + + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { + //Store the deleted item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyDeletedItem(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store deleted affiliation in panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); } /** @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setClientSourceId(clientSourceId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } } @Override @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setOrcid(orcid); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } + } @Override diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index d57dde50df1..554d47b39a6 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,10 +457,29 @@ + + + + + + + + + + + + + + + + - + + + + diff --git a/orcid-test/src/main/resources/properties/test-api-common.properties b/orcid-test/src/main/resources/properties/test-api-common.properties index 537759cfde2..dc4889fc41d 100644 --- a/orcid-test/src/main/resources/properties/test-api-common.properties +++ b/orcid-test/src/main/resources/properties/test-api-common.properties @@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-api-internal.properties b/orcid-test/src/main/resources/properties/test-api-internal.properties index d85df6f3aba..284325b6265 100644 --- a/orcid-test/src/main/resources/properties/test-api-internal.properties +++ b/orcid-test/src/main/resources/properties/test-api-internal.properties @@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-core.properties b/orcid-test/src/main/resources/properties/test-core.properties index ab3c0577d4c..1dc1d3e9298 100644 --- a/orcid-test/src/main/resources/properties/test-core.properties +++ b/orcid-test/src/main/resources/properties/test-core.properties @@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-db.properties b/orcid-test/src/main/resources/properties/test-db.properties index 8f8e05d0d68..a2c679f182a 100644 --- a/orcid-test/src/main/resources/properties/test-db.properties +++ b/orcid-test/src/main/resources/properties/test-db.properties @@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true org.orcid.persistence.togglz.db.preferredTestQuery=select 1 org.orcid.persistence.togglz.db.numHelperThreads=5 org.orcid.persistence.togglz.cache.ttl=60000 + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties index 10d29640a51..f1d06636692 100644 --- a/orcid-test/src/main/resources/properties/test-scheduler.properties +++ b/orcid-test/src/main/resources/properties/test-scheduler.properties @@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-ui.properties b/orcid-test/src/main/resources/properties/test-ui.properties index 6784fbe3068..9741a3fc49b 100644 --- a/orcid-test/src/main/resources/properties/test-ui.properties +++ b/orcid-test/src/main/resources/properties/test-ui.properties @@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index d7b361ba22d..6ee5bc45646 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-context + + org.springframework + spring-jdbc + org.apache.commons commons-lang3 @@ -113,6 +117,22 @@ jsoup 1.15.4 + + + + com.amazon.redshift + redshift-jdbc42 + 2.1.0.30 + + + + + + com.zaxxer + HikariCP + 5.0.1 + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java new file mode 100644 index 00000000000..378b0185227 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -0,0 +1,55 @@ +package org.orcid.utils.panoply; + +public class PanoplyDeletedItem { + private Long id; + private String dwTable; + private Long itemId; + private String clientSourceId; + private String orcid; + + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; + public final String DW_WORK = "dw_work"; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getDwTable() { + return dwTable; + } + public void setDwTable(String dwTable) { + this.dwTable = dwTable; + } + public Long getItemId() { + return itemId; + } + public void setItemId(Long itemId) { + this.itemId = itemId; + } + public String getClientSourceId() { + return clientSourceId; + } + public void setClientSourceId(String clientSourceId) { + this.clientSourceId = clientSourceId; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + + @Override + public String toString() { + return "PanoplyDeletedItem{" + + "id=" + id + + ", dwTable='" + dwTable + '\'' + + ", itemId='" + itemId + '\'' + + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + + '}'; + } + +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java new file mode 100644 index 00000000000..3499593eb27 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -0,0 +1,36 @@ +package org.orcid.utils.panoply; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + + +import java.util.Date; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.orcid.utils.alerting.SlackManager; + +@Repository +public class PanoplyRedshiftClient { + + private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); + + @Autowired + @Qualifier("panoplyJdbcTemplate") + private JdbcTemplate panoplyJdbcTemplate; + + + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); + } + +} diff --git a/properties/development.properties b/properties/development.properties index 34eae804933..9186b22ceb8 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -#org.orcid.persistence.liquibase.enabled=false +org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx