Merge branch 'main' into add-verification-date-to-email-endpoint

ORCID · Oct 15, 2024 · bfdc0ee · bfdc0ee
2 parents 821877e + 14d1885
commit bfdc0ee
Show file tree

Hide file tree

Showing 13 changed files with 294 additions and 11 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,11 @@
+## v2.66.4 - 2024-10-11
+
+[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.3...v2.66.4)
+
+## v2.66.3 - 2024-10-11
+
+[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.2...v2.66.3)
+
 ## v2.66.2 - 2024-10-10
 
 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.1...v2.66.2)

diff --git a/...rsistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/...rsistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java
@@ -3,19 +3,31 @@
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
+import javax.annotation.Resource;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
 
+import org.orcid.utils.panoply.PanoplyDeletedItem;
+import org.orcid.utils.panoply.PanoplyRedshiftClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.orcid.persistence.aop.UpdateProfileLastModified;
 import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
 import org.orcid.persistence.dao.OrgAffiliationRelationDao;
 import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.transaction.annotation.Transactional;
 
 public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliationRelationEntity, Long> implements OrgAffiliationRelationDao {
 
+    private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class);
+
+    @Value("${org.orcid.persistence.panoply.cleanup.production:false}")
+    private boolean enablePanoplyCleanupInProduction;
+
     private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION";
 
     private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION";
@@ -30,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliation
 
     private static final String AFFILIATION_TYPE_SERVICE = "SERVICE";
 
+    private static final String DW_PANOPLY_AFFILIATION_TABLE = "dw_org_affiliation_relation";
+
+    @Resource
+    private PanoplyRedshiftClient panoplyClient;
+
     public OrgAffiliationRelationDaoImpl() {
         super(OrgAffiliationRelationEntity.class);
     }
@@ -51,7 +68,34 @@ public boolean removeOrgAffiliationRelation(String userOrcid, Long orgAffiliatio
         Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid=:userOrcid and id=:orgAffiliationRelationId");
         query.setParameter("userOrcid", userOrcid);
         query.setParameter("orgAffiliationRelationId", orgAffiliationRelationId);
-        return query.executeUpdate() > 0 ? true : false;
+        if (query.executeUpdate() > 0) {
+            if (enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setItemId(orgAffiliationRelationId);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private void storeDeletedItemInPanoply(PanoplyDeletedItem item) {
+        // Store the deleted item in panoply Db without blocking
+        CompletableFuture.supplyAsync(() -> {
+            try {
+                panoplyClient.addPanoplyDeletedItem(item);
+                return true;
+            } catch (Exception e) {
+                LOG.error("Cannot store deleted affiliation in panoply ", e);
+                return false;
+            }
+        }).thenAccept(result -> {
+            if (!result) {
+                LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result);
+            }
+
+        });
     }
 
     /**
@@ -196,6 +240,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) {
         Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId");
         query.setParameter("clientSourceId", clientSourceId);
         query.executeUpdate();
+        if (query.executeUpdate() > 0) {
+            if (enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setClientSourceId(clientSourceId);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+            }
+        }
     }
 
     @Override
@@ -281,6 +333,15 @@ public void removeAllAffiliations(String orcid) {
         Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid");
         query.setParameter("orcid", orcid);
         query.executeUpdate();
+        if (query.executeUpdate() > 0) {
+            if (enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setOrcid(orcid);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+            }
+        }
+
     }
 
     @Override
@@ -385,8 +446,7 @@ public void revertUserOBODetails(List<BigInteger> ids) {
     @SuppressWarnings("unchecked")
     @Override
     public List<BigInteger> getIdsForUserOBORecords(int max) {
-        Query query = entityManager
-                .createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL");
+        Query query = entityManager.createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL");
         query.setMaxResults(max);
         return query.getResultList();
     }
@@ -399,14 +459,14 @@ public List<BigInteger> getIdsOfOrgAffiliationRelationsReferencingClientProfiles
         query.setMaxResults(max);
         return query.getResultList();
     }
-    
+
     @Override
     @UpdateProfileLastModifiedAndIndexingStatus
     @Transactional
     public void persist(OrgAffiliationRelationEntity affiliation) {
         super.persist(affiliation);
     }
-    
+
     @Override
     @UpdateProfileLastModifiedAndIndexingStatus
     @Transactional

diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml
@@ -457,10 +457,29 @@
         <property name="entityManager" ref="entityManagerReadOnly" />
     </bean>
 
+
+    <!-- Panoply Redshift -->
+	<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
+		<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver:com.amazon.redshift.jdbc.Driver}" />
+        <property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
+        <property name="username" value="\${org.orcid.core.utils.panoply.username}" />
+        <property name="password" value="\${org.orcid.core.utils.panoply.password}" />
+        <property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize:5}" />
+        <property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout:3600}" />
+        <property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout:3600}" />
+	</bean>
+
+	<!-- Panoply JdbcTemplate Bean Definition -->
+    <bean id="panoplyJdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
+        <property name="dataSource" ref="panoplyRedshiftDataSource" />
+    </bean>
 
     <!-- Statistics -->
     <bean id="statisticsDao" class="org.orcid.persistence.dao.impl.StatisticsDaoImpl">
         <property name="entityManager" ref="entityManager" />
-    </bean>     
+    </bean>   
+
+    <!-- Redshift panoply -->
+    <bean id="panoplyClient" class="org.orcid.utils.panoply.PanoplyRedshiftClient" />  
 
 </beans>
diff --git a/orcid-test/src/main/resources/properties/test-api-common.properties b/orcid-test/src/main/resources/properties/test-api-common.properties
@@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-test/src/main/resources/properties/test-api-internal.properties b/orcid-test/src/main/resources/properties/test-api-internal.properties
@@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-test/src/main/resources/properties/test-core.properties b/orcid-test/src/main/resources/properties/test-core.properties
@@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-test/src/main/resources/properties/test-db.properties b/orcid-test/src/main/resources/properties/test-db.properties
@@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true
 org.orcid.persistence.togglz.db.preferredTestQuery=select 1
 org.orcid.persistence.togglz.db.numHelperThreads=5
 org.orcid.persistence.togglz.cache.ttl=60000
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties
@@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
 org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-test/src/main/resources/properties/test-ui.properties b/orcid-test/src/main/resources/properties/test-ui.properties
@@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx
diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml
@@ -43,6 +43,10 @@
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-context</artifactId>
 		</dependency>
+		<dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-lang3</artifactId>
@@ -113,6 +117,22 @@
 			<artifactId>jsoup</artifactId>
 			<version>1.15.4</version>
 		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 -->
+		<dependency>
+		    <groupId>com.amazon.redshift</groupId>
+		    <artifactId>redshift-jdbc42</artifactId>
+		    <version>2.1.0.30</version>
+		</dependency>
+
+
+		<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
+		<dependency>
+		    <groupId>com.zaxxer</groupId>
+		    <artifactId>HikariCP</artifactId>
+		    <version>5.0.1</version>
+		</dependency>
+
 
 		<!-- Test dependencies -->
 

diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java
@@ -0,0 +1,59 @@
+package org.orcid.utils.panoply;
+
+public class PanoplyDeletedItem {
+    private Long id;
+    private String dwTable;
+    private Long itemId;
+    private String clientSourceId;
+    private String orcid;
+
+    public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation";
+    public final String DW_WORK = "dw_work";
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public String getDwTable() {
+        return dwTable;
+    }
+
+    public void setDwTable(String dwTable) {
+        this.dwTable = dwTable;
+    }
+
+    public Long getItemId() {
+        return itemId;
+    }
+
+    public void setItemId(Long itemId) {
+        this.itemId = itemId;
+    }
+
+    public String getClientSourceId() {
+        return clientSourceId;
+    }
+
+    public void setClientSourceId(String clientSourceId) {
+        this.clientSourceId = clientSourceId;
+    }
+
+    public String getOrcid() {
+        return orcid;
+    }
+
+    public void setOrcid(String orcid) {
+        this.orcid = orcid;
+    }
+
+    @Override
+    public String toString() {
+        return "PanoplyDeletedItem{" + "id=" + id + ", dwTable='" + dwTable + '\'' + ", itemId='" + itemId + '\'' + ", clientSourceId='" + clientSourceId + '\''
+                + ", orcid='" + orcid + '\'' + '}';
+    }
+
+}