Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table""

orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java

@@ -3,19 +3,32 @@
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.CompletableFuture;
 
+import javax.annotation.Resource;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
 
+import org.orcid.utils.panoply.PanoplyDeletedItem;
+import org.orcid.utils.panoply.PanoplyRedshiftClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.commons.logging.Log;
 import org.orcid.persistence.aop.UpdateProfileLastModified;
 import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
 import org.orcid.persistence.dao.OrgAffiliationRelationDao;
 import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.cache.annotation.Cacheable;
 import org.springframework.transaction.annotation.Transactional;
 
 public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliationRelationEntity, Long> implements OrgAffiliationRelationDao {
-
+
+    private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class);
+
+    @Value("${org.orcid.persistence.panoply.cleanup.production:false}")
+    private boolean enablePanoplyCleanupInProduction; 
+
     private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION";
 
     private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION";
@@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliation
     private static final String AFFILIATION_TYPE_QUALIFICATION = "QUALIFICATION";
 
     private static final String AFFILIATION_TYPE_SERVICE = "SERVICE";
+
+    private static final String DW_PANOPLY_AFFILIATION_TABLE = "dw_org_affiliation_relation";
+
+    @Resource
+    private PanoplyRedshiftClient panoplyClient;  
 
     public OrgAffiliationRelationDaoImpl() {
         super(OrgAffiliationRelationEntity.class);
@@ -51,7 +69,34 @@ public boolean removeOrgAffiliationRelation(String userOrcid, Long orgAffiliatio
         Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid=:userOrcid and id=:orgAffiliationRelationId");
         query.setParameter("userOrcid", userOrcid);
         query.setParameter("orgAffiliationRelationId", orgAffiliationRelationId);
-        return query.executeUpdate() > 0 ? true : false;
+        if(query.executeUpdate() > 0) {
+            if(enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setItemId(orgAffiliationRelationId);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+             }
+            return true;
+        }
+        return   false;
+    }
+
+    private void storeDeletedItemInPanoply(PanoplyDeletedItem item) {
+        //Store the deleted item in panoply Db without blocking
+        CompletableFuture.supplyAsync(() -> {
+            try {
+                panoplyClient.addPanoplyDeletedItem(item);
+                return true;
+            } catch (Exception e) {
+                LOG.error("Cannot store deleted affiliation in panoply ", e);
+                return false;
+            }
+        }).thenAccept(result -> {            
+            if(! result) {
+                LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result);
+            }
+
+        });
     }
 
     /**
@@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) {
         Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId");
         query.setParameter("clientSourceId", clientSourceId);
         query.executeUpdate();
+        if(query.executeUpdate() > 0) {
+            if(enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setClientSourceId(clientSourceId);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+             }
+        }
     }
 
     @Override
@@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) {
         Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid");
         query.setParameter("orcid", orcid);
         query.executeUpdate();
+        if(query.executeUpdate() > 0) {
+            if(enablePanoplyCleanupInProduction) {
+                PanoplyDeletedItem item = new PanoplyDeletedItem();
+                item.setOrcid(orcid);
+                item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
+                storeDeletedItemInPanoply(item);
+             }
+        }
+
     }
 
     @Override

orcid-persistence/src/main/resources/orcid-persistence-context.xml

@@ -457,10 +457,29 @@
         <property name="entityManager" ref="entityManagerReadOnly" />
     </bean>
 
+
+    <!-- Panoply Redshift -->
+	<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
+		<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver}" />
+        <property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
+        <property name="username" value="\${org.orcid.core.utils.panoply.username}" />
+        <property name="password" value="\${org.orcid.core.utils.panoply.password}" />
+        <property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize}" />
+        <property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout}" />
+        <property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout}" />
+	</bean>
+
+	<!-- Panoply JdbcTemplate Bean Definition -->
+    <bean id="panoplyJdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
+        <property name="dataSource" ref="panoplyRedshiftDataSource" />
+    </bean>
 
     <!-- Statistics -->
     <bean id="statisticsDao" class="org.orcid.persistence.dao.impl.StatisticsDaoImpl">
         <property name="entityManager" ref="entityManager" />
-    </bean>     
+    </bean>   
+
+    <!-- Redshift panoply -->
+    <bean id="panoplyClient" class="org.orcid.utils.panoply.PanoplyRedshiftClient" />  
 
 </beans>

orcid-test/src/main/resources/properties/test-api-common.properties

@@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-test/src/main/resources/properties/test-api-internal.properties

@@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-test/src/main/resources/properties/test-core.properties

@@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-test/src/main/resources/properties/test-db.properties

@@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true
 org.orcid.persistence.togglz.db.preferredTestQuery=select 1
 org.orcid.persistence.togglz.db.numHelperThreads=5
 org.orcid.persistence.togglz.cache.ttl=60000
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-test/src/main/resources/properties/test-scheduler.properties

@@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
 org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-test/src/main/resources/properties/test-ui.properties

@@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10
 # Redis
 org.orcid.core.utils.cache.redis.host=xxx.xxx.com
 org.orcid.core.utils.cache.redis.port=6379
-org.orcid.core.utils.cache.redis.password=XXXX
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.core.utils.cache.redis.password=XXXX
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx

orcid-utils/pom.xml

@@ -43,6 +43,10 @@
 			<groupId>org.springframework</groupId>
 			<artifactId>spring-context</artifactId>
 		</dependency>
+		<dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-jdbc</artifactId>
+        </dependency>
 		<dependency>
 			<groupId>org.apache.commons</groupId>
 			<artifactId>commons-lang3</artifactId>
@@ -113,6 +117,22 @@
 			<artifactId>jsoup</artifactId>
 			<version>1.15.4</version>
 		</dependency>
+
+		<!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 -->
+		<dependency>
+		    <groupId>com.amazon.redshift</groupId>
+		    <artifactId>redshift-jdbc42</artifactId>
+		    <version>2.1.0.30</version>
+		</dependency>
+
+
+		<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
+		<dependency>
+		    <groupId>com.zaxxer</groupId>
+		    <artifactId>HikariCP</artifactId>
+		    <version>5.0.1</version>
+		</dependency>
+
 
 		<!-- Test dependencies -->
 

orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java

@@ -0,0 +1,55 @@
+package org.orcid.utils.panoply;
+
+public class PanoplyDeletedItem {
+    private Long id;
+    private String dwTable;
+    private Long itemId;
+    private String clientSourceId;
+    private String orcid;
+
+    public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation";
+    public final String DW_WORK = "dw_work";
+
+    public Long getId() {
+        return id;
+    }
+    public void setId(Long id) {
+        this.id = id;
+    }
+    public String getDwTable() {
+        return dwTable;
+    }
+    public void setDwTable(String dwTable) {
+        this.dwTable = dwTable;
+    }
+    public Long getItemId() {
+        return itemId;
+    }
+    public void setItemId(Long itemId) {
+        this.itemId = itemId;
+    }
+    public String getClientSourceId() {
+        return clientSourceId;
+    }
+    public void setClientSourceId(String clientSourceId) {
+        this.clientSourceId = clientSourceId;
+    }
+    public String getOrcid() {
+        return orcid;
+    }
+    public void setOrcid(String orcid) {
+        this.orcid = orcid;
+    }
+
+    @Override
+    public String toString() {
+        return "PanoplyDeletedItem{" +
+                "id=" + id +
+                ", dwTable='" + dwTable + '\'' +
+                ", itemId='" + itemId + '\'' +
+                ", clientSourceId='" + clientSourceId + '\'' +
+                ", orcid='" + orcid + '\'' +
+                '}';
+    }
+
+}

orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java

@@ -0,0 +1,36 @@
+package org.orcid.utils.panoply;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.stereotype.Repository;
+import org.springframework.stereotype.Service;
+
+
+import java.util.Date;
+
+import javax.annotation.Resource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.orcid.utils.alerting.SlackManager;
+
+@Repository
+public class PanoplyRedshiftClient {
+
+    private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class);
+
+    @Autowired
+    @Qualifier("panoplyJdbcTemplate")
+    private JdbcTemplate panoplyJdbcTemplate;
+
+
+    public int addPanoplyDeletedItem(PanoplyDeletedItem item) {
+        LOG.debug("Adding deleted item to panoply DB: " +  item.toString());
+        String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)";
+        return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable());
+    }
+
+}

properties/development.properties

@@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true
 org.orcid.core.autospam.slackChannel=collab-spam-reports
 org.orcid.core.autospam.webhookUrl=
 
-#org.orcid.persistence.liquibase.enabled=false
+org.orcid.persistence.liquibase.enabled=false
 org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr
+
+org.orcid.persistence.panoply.cleanup.production=false
+# Panoply redshift database 
+org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
+org.orcid.core.utils.panoply.maxPoolSize=20
+org.orcid.core.utils.panoply.password=xxx
+org.orcid.core.utils.panoply.idleConnectionTimeout=3600
+org.orcid.core.utils.panoply.connectionTimeout=36000
+org.orcid.core.utils.panoply.jdbcUrl=xxx
+org.orcid.core.utils.panoply.username=xxx