Skip to content

Commit

Permalink
Revert "Revert "Added the code to store deleted items in panoply dw_d…
Browse files Browse the repository at this point in the history
…eleted_items table"" (#7100)

* Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table""

* Fixed the failures in test suite

---------

Co-authored-by: Camelia Dumitru <[email protected]>
  • Loading branch information
amontenegro and Camelia-Orcid authored Oct 11, 2024
1 parent 2be3701 commit 6a07535
Show file tree
Hide file tree
Showing 12 changed files with 282 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,32 @@
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;

import javax.annotation.Resource;
import javax.persistence.Query;
import javax.persistence.TypedQuery;

import org.orcid.utils.panoply.PanoplyDeletedItem;
import org.orcid.utils.panoply.PanoplyRedshiftClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.logging.Log;
import org.orcid.persistence.aop.UpdateProfileLastModified;
import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
import org.orcid.persistence.dao.OrgAffiliationRelationDao;
import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.transaction.annotation.Transactional;

public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliationRelationEntity, Long> implements OrgAffiliationRelationDao {


private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class);

@Value("${org.orcid.persistence.panoply.cleanup.production:false}")
private boolean enablePanoplyCleanupInProduction;

private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION";

private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION";
Expand All @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliation
private static final String AFFILIATION_TYPE_QUALIFICATION = "QUALIFICATION";

private static final String AFFILIATION_TYPE_SERVICE = "SERVICE";

private static final String DW_PANOPLY_AFFILIATION_TABLE = "dw_org_affiliation_relation";

@Resource
private PanoplyRedshiftClient panoplyClient;

public OrgAffiliationRelationDaoImpl() {
super(OrgAffiliationRelationEntity.class);
Expand All @@ -51,7 +69,34 @@ public boolean removeOrgAffiliationRelation(String userOrcid, Long orgAffiliatio
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid=:userOrcid and id=:orgAffiliationRelationId");
query.setParameter("userOrcid", userOrcid);
query.setParameter("orgAffiliationRelationId", orgAffiliationRelationId);
return query.executeUpdate() > 0 ? true : false;
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setItemId(orgAffiliationRelationId);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
return true;
}
return false;
}

private void storeDeletedItemInPanoply(PanoplyDeletedItem item) {
//Store the deleted item in panoply Db without blocking
CompletableFuture.supplyAsync(() -> {
try {
panoplyClient.addPanoplyDeletedItem(item);
return true;
} catch (Exception e) {
LOG.error("Cannot store deleted affiliation in panoply ", e);
return false;
}
}).thenAccept(result -> {
if(! result) {
LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result);
}

});
}

/**
Expand Down Expand Up @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) {
Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId");
query.setParameter("clientSourceId", clientSourceId);
query.executeUpdate();
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setClientSourceId(clientSourceId);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
}
}

@Override
Expand Down Expand Up @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) {
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid");
query.setParameter("orcid", orcid);
query.executeUpdate();
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setOrcid(orcid);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
}

}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -457,10 +457,29 @@
<property name="entityManager" ref="entityManagerReadOnly" />
</bean>


<!-- Panoply Redshift -->
<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver}" />
<property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="\${org.orcid.core.utils.panoply.username}" />
<property name="password" value="\${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize}" />
<property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout}" />
<property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout}" />
</bean>

<!-- Panoply JdbcTemplate Bean Definition -->
<bean id="panoplyJdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="panoplyRedshiftDataSource" />
</bean>

<!-- Statistics -->
<bean id="statisticsDao" class="org.orcid.persistence.dao.impl.StatisticsDaoImpl">
<property name="entityManager" ref="entityManager" />
</bean>
</bean>

<!-- Redshift panoply -->
<bean id="panoplyClient" class="org.orcid.utils.panoply.PanoplyRedshiftClient" />

</beans>
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize
# Redis
org.orcid.core.utils.cache.redis.host=xxx.xxx.com
org.orcid.core.utils.cache.redis.port=6379
org.orcid.core.utils.cache.redis.password=XXXX
org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3
# Redis
org.orcid.core.utils.cache.redis.host=xxx.xxx.com
org.orcid.core.utils.cache.redis.port=6379
org.orcid.core.utils.cache.redis.password=XXXX
org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
14 changes: 13 additions & 1 deletion orcid-test/src/main/resources/properties/test-core.properties
Original file line number Diff line number Diff line change
Expand Up @@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer
# Redis
org.orcid.core.utils.cache.redis.host=xxx.xxx.com
org.orcid.core.utils.cache.redis.port=6379
org.orcid.core.utils.cache.redis.password=XXXX
org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
12 changes: 12 additions & 0 deletions orcid-test/src/main/resources/properties/test-db.properties
Original file line number Diff line number Diff line change
Expand Up @@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true
org.orcid.persistence.togglz.db.preferredTestQuery=select 1
org.orcid.persistence.togglz.db.numHelperThreads=5
org.orcid.persistence.togglz.cache.ttl=60000

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
12 changes: 12 additions & 0 deletions orcid-test/src/main/resources/properties/test-scheduler.properties
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip
org.orcid.core.utils.cache.redis.host=xxx.xxx.com
org.orcid.core.utils.cache.redis.port=6379
org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
14 changes: 13 additions & 1 deletion orcid-test/src/main/resources/properties/test-ui.properties
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10
# Redis
org.orcid.core.utils.cache.redis.host=xxx.xxx.com
org.orcid.core.utils.cache.redis.port=6379
org.orcid.core.utils.cache.redis.password=XXXX
org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.core.utils.cache.redis.password=XXXX

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx
20 changes: 20 additions & 0 deletions orcid-utils/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand Down Expand Up @@ -113,6 +117,22 @@
<artifactId>jsoup</artifactId>
<version>1.15.4</version>
</dependency>

<!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 -->
<dependency>
<groupId>com.amazon.redshift</groupId>
<artifactId>redshift-jdbc42</artifactId>
<version>2.1.0.30</version>
</dependency>


<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>5.0.1</version>
</dependency>


<!-- Test dependencies -->

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.orcid.utils.panoply;

public class PanoplyDeletedItem {
private Long id;
private String dwTable;
private Long itemId;
private String clientSourceId;
private String orcid;

public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation";
public final String DW_WORK = "dw_work";

public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getDwTable() {
return dwTable;
}
public void setDwTable(String dwTable) {
this.dwTable = dwTable;
}
public Long getItemId() {
return itemId;
}
public void setItemId(Long itemId) {
this.itemId = itemId;
}
public String getClientSourceId() {
return clientSourceId;
}
public void setClientSourceId(String clientSourceId) {
this.clientSourceId = clientSourceId;
}
public String getOrcid() {
return orcid;
}
public void setOrcid(String orcid) {
this.orcid = orcid;
}

@Override
public String toString() {
return "PanoplyDeletedItem{" +
"id=" + id +
", dwTable='" + dwTable + '\'' +
", itemId='" + itemId + '\'' +
", clientSourceId='" + clientSourceId + '\'' +
", orcid='" + orcid + '\'' +
'}';
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package org.orcid.utils.panoply;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.stereotype.Service;


import java.util.Date;

import javax.annotation.Resource;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.orcid.utils.alerting.SlackManager;

@Repository
public class PanoplyRedshiftClient {

private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class);

@Autowired
@Qualifier("panoplyJdbcTemplate")
private JdbcTemplate panoplyJdbcTemplate;


public int addPanoplyDeletedItem(PanoplyDeletedItem item) {
LOG.debug("Adding deleted item to panoply DB: " + item.toString());
String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)";
return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable());
}

}
12 changes: 11 additions & 1 deletion properties/development.properties
Original file line number Diff line number Diff line change
Expand Up @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true
org.orcid.core.autospam.slackChannel=collab-spam-reports
org.orcid.core.autospam.webhookUrl=

#org.orcid.persistence.liquibase.enabled=false
org.orcid.persistence.liquibase.enabled=false
org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx

0 comments on commit 6a07535

Please sign in to comment.