Skip to content

Commit

Permalink
Added the code to store deleted items in panoply dw_deleted_items table
Browse files Browse the repository at this point in the history
The items will be processed by the script that runs daily and cleaned up after the item is deleted from the corresponding view
  • Loading branch information
Camelia-Orcid committed Sep 27, 2024
1 parent d749f3a commit a9da77d
Show file tree
Hide file tree
Showing 6 changed files with 202 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,32 @@
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;

import javax.annotation.Resource;
import javax.persistence.Query;
import javax.persistence.TypedQuery;

import org.orcid.utils.panoply.PanoplyDeletedItem;
import org.orcid.utils.panoply.PanoplyRedshiftClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.commons.logging.Log;
import org.orcid.persistence.aop.UpdateProfileLastModified;
import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
import org.orcid.persistence.dao.OrgAffiliationRelationDao;
import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.cache.annotation.Cacheable;
import org.springframework.transaction.annotation.Transactional;

public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliationRelationEntity, Long> implements OrgAffiliationRelationDao {


private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class);

@Value("${org.orcid.persistence.panoply.cleanup.production:false}")
private boolean enablePanoplyCleanupInProduction;

private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION";

private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION";
Expand All @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl<OrgAffiliation
private static final String AFFILIATION_TYPE_QUALIFICATION = "QUALIFICATION";

private static final String AFFILIATION_TYPE_SERVICE = "SERVICE";

private static final String DW_PANOPLY_AFFILIATION_TABLE = "dw_org_affiliation_relation";

@Resource
private PanoplyRedshiftClient panoplyClient;

public OrgAffiliationRelationDaoImpl() {
super(OrgAffiliationRelationEntity.class);
Expand All @@ -51,7 +69,34 @@ public boolean removeOrgAffiliationRelation(String userOrcid, Long orgAffiliatio
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid=:userOrcid and id=:orgAffiliationRelationId");
query.setParameter("userOrcid", userOrcid);
query.setParameter("orgAffiliationRelationId", orgAffiliationRelationId);
return query.executeUpdate() > 0 ? true : false;
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setItemId(orgAffiliationRelationId);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
return true;
}
return false;
}

private void storeDeletedItemInPanoply(PanoplyDeletedItem item) {
//Store the deleted item in panoply Db without blocking
CompletableFuture.supplyAsync(() -> {
try {
panoplyClient.addPanoplyDeletedItem(item);
return true;
} catch (Exception e) {
LOG.error("Cannot store deleted affiliation in panoply ", e);
return false;
}
}).thenAccept(result -> {
if(! result) {
LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result);
}

});
}

/**
Expand Down Expand Up @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) {
Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId");
query.setParameter("clientSourceId", clientSourceId);
query.executeUpdate();
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setClientSourceId(clientSourceId);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
}
}

@Override
Expand Down Expand Up @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) {
Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid");
query.setParameter("orcid", orcid);
query.executeUpdate();
if(query.executeUpdate() > 0) {
if(enablePanoplyCleanupInProduction) {
PanoplyDeletedItem item = new PanoplyDeletedItem();
item.setOrcid(orcid);
item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE);
storeDeletedItemInPanoply(item);
}
}

}

@Override
Expand Down
16 changes: 16 additions & 0 deletions orcid-persistence/src/main/resources/orcid-persistence-context.xml
Original file line number Diff line number Diff line change
Expand Up @@ -457,6 +457,22 @@
<property name="entityManager" ref="entityManagerReadOnly" />
</bean>


<!-- Panoply Redshift -->
<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver}" />
<property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="\${org.orcid.core.utils.panoply.username}" />
<property name="password" value="\${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize}" />
<property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout}" />
<property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout}" />
</bean>

<!-- Panoply JdbcTemplate Bean Definition -->
<bean id="panoplyJdbcTemplate" class="org.springframework.jdbc.core.JdbcTemplate">
<property name="dataSource" ref="panoplyRedshiftDataSource" />
</bean>

<!-- Statistics -->
<bean id="statisticsDao" class="org.orcid.persistence.dao.impl.StatisticsDaoImpl">
Expand Down
20 changes: 20 additions & 0 deletions orcid-utils/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@
<groupId>org.springframework</groupId>
<artifactId>spring-context</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
Expand Down Expand Up @@ -113,6 +117,22 @@
<artifactId>jsoup</artifactId>
<version>1.15.4</version>
</dependency>

<!-- https://mvnrepository.com/artifact/com.amazon.redshift/redshift-jdbc42 -->
<dependency>
<groupId>com.amazon.redshift</groupId>
<artifactId>redshift-jdbc42</artifactId>
<version>2.1.0.30</version>
</dependency>


<!-- https://mvnrepository.com/artifact/com.zaxxer/HikariCP -->
<dependency>
<groupId>com.zaxxer</groupId>
<artifactId>HikariCP</artifactId>
<version>5.0.1</version>
</dependency>


<!-- Test dependencies -->

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
package org.orcid.utils.panoply;

public class PanoplyDeletedItem {
private Long id;
private String dwTable;
private Long itemId;
private String clientSourceId;
private String orcid;

public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation";
public final String DW_WORK = "dw_work";

public Long getId() {
return id;
}
public void setId(Long id) {
this.id = id;
}
public String getDwTable() {
return dwTable;
}
public void setDwTable(String dwTable) {
this.dwTable = dwTable;
}
public Long getItemId() {
return itemId;
}
public void setItemId(Long itemId) {
this.itemId = itemId;
}
public String getClientSourceId() {
return clientSourceId;
}
public void setClientSourceId(String clientSourceId) {
this.clientSourceId = clientSourceId;
}
public String getOrcid() {
return orcid;
}
public void setOrcid(String orcid) {
this.orcid = orcid;
}

@Override
public String toString() {
return "PanoplyDeletedItem{" +
"id=" + id +
", dwTable='" + dwTable + '\'' +
", itemId='" + itemId + '\'' +
", clientSourceId='" + clientSourceId + '\'' +
", orcid='" + orcid + '\'' +
'}';
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package org.orcid.utils.panoply;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Repository;
import org.springframework.stereotype.Service;


import java.util.Date;

import javax.annotation.Resource;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.orcid.utils.alerting.SlackManager;

@Repository
public class PanoplyRedshiftClient {

private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class);

@Autowired
@Qualifier("panoplyJdbcTemplate")
private JdbcTemplate panoplyJdbcTemplate;


public int addPanoplyDeletedItem(PanoplyDeletedItem item) {
LOG.debug("Adding deleted item to panoply DB: " + item.toString());
String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)";
return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable());
}

}
12 changes: 11 additions & 1 deletion properties/development.properties
Original file line number Diff line number Diff line change
Expand Up @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true
org.orcid.core.autospam.slackChannel=collab-spam-reports
org.orcid.core.autospam.webhookUrl=

#org.orcid.persistence.liquibase.enabled=false
org.orcid.persistence.liquibase.enabled=false
org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr

org.orcid.persistence.panoply.cleanup.production=false
# Panoply redshift database
org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver
org.orcid.core.utils.panoply.maxPoolSize=20
org.orcid.core.utils.panoply.password=xxx
org.orcid.core.utils.panoply.idleConnectionTimeout=3600
org.orcid.core.utils.panoply.connectionTimeout=36000
org.orcid.core.utils.panoply.jdbcUrl=xxx
org.orcid.core.utils.panoply.username=xxx

0 comments on commit a9da77d

Please sign in to comment.