From 4912303363cfc25506a437ca09a73482b3554d06 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Mon, 23 Sep 2024 22:22:22 +0100 Subject: [PATCH 01/18] Email domains fixes for summary endpoint (#7089) * Email domains fixes for summary endpoint * Fixes for summary unit test for marshalling --------- Co-authored-by: Angel Montenegro --- .../manager/impl/SummaryManagerImpl.java | 10 +++---- .../org/orcid/core/model/EmailDomain.java | 11 +++++--- .../pojo/summary/EmailDomainSummary.java | 26 ------------------- 3 files changed, 12 insertions(+), 35 deletions(-) diff --git a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java index 766f3f4f65a..3ecd48fb678 100644 --- a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java @@ -73,10 +73,11 @@ import org.orcid.pojo.summary.ExternalIdentifiersSummary; import org.orcid.pojo.summary.RecordSummaryPojo; import org.orcid.utils.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; public class SummaryManagerImpl implements SummaryManager { - @Resource(name = "recordNameManagerReadOnlyV3") private RecordNameManagerReadOnly recordNameManagerReadOnly; @@ -285,8 +286,7 @@ public RecordSummaryPojo getRecordSummaryPojo(String orcid) { for (EmailDomain ed : recordSummary.getEmailDomains().getEmailDomains()) { EmailDomainSummary eds = new EmailDomainSummary(); eds.setValue(ed.getValue()); - eds.setCreatedDate(ed.getCreatedDate().toFuzzyDate().toString()); - eds.setLastModified(ed.getLastModified().toFuzzyDate().toString()); + emailDomains.add(eds); } } pojo.setEmailDomains(emailDomains); @@ -520,7 +520,6 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci emailDomains = profileEmailDomainManagerReadOnly.getPublicEmailDomains(orcid); List edList = new ArrayList(); if (emailDomains != null && !emailDomains.isEmpty()) { - EmailDomain ed = null; for (ProfileEmailDomainEntity ped : emailDomains) { ed = new EmailDomain(); @@ -530,7 +529,6 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci edList.add(ed); } } - List emailDomainsTop3 = new ArrayList(); edList.stream().limit(3).forEach(t -> { EmailDomain ed = new EmailDomain(); @@ -544,7 +542,9 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci eds.setCount(edList.size()); if (!emailDomainsTop3.isEmpty()) { eds.setEmailDomains(emailDomainsTop3); + } + recordSummary.setEmailDomains(eds); } } diff --git a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java index a31928c6841..779717dd90d 100644 --- a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java +++ b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java @@ -6,6 +6,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; import org.orcid.jaxb.model.v3.release.common.FuzzyDate; @@ -14,9 +15,9 @@ import io.swagger.v3.oas.annotations.media.Schema; @XmlAccessorType(XmlAccessType.FIELD) -@XmlType(propOrder = { "value","createdDate", "lastModified"}) -@XmlRootElement(name = "education-qualification", namespace = "http://www.orcid.org/ns/summary") -@Schema(description = "Education Qualification") +@XmlType(propOrder = { "value", "createdDate","lastModified"}) +@XmlRootElement(name = "email-domain", namespace = "http://www.orcid.org/ns/summary") +@Schema(description = "Email Domain") public class EmailDomain { @XmlElement(name = "value", namespace = "http://www.orcid.org/ns/summary") protected String value; @@ -32,7 +33,8 @@ public String getValue() { public void setValue(String value) { this.value = value; } - + + @XmlTransient public Date getCreatedDate() { return createdDate; } @@ -41,6 +43,7 @@ public void setCreatedDate(Date createdDate) { this.createdDate = createdDate; } + @XmlTransient public Date getLastModified() { return lastModified; } diff --git a/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java b/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java index 4513d3db408..a8e6c1ba0a8 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java +++ b/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java @@ -5,8 +5,6 @@ public class EmailDomainSummary { private String value; - private String createdDate; - private String lastModified; public String getValue() { return value; @@ -16,22 +14,6 @@ public void setValue(String value) { this.value = value; } - public String getCreatedDate() { - return createdDate; - } - - public void setCreatedDate(String createdDate) { - this.createdDate = createdDate; - } - - public String getLastModified() { - return lastModified; - } - - public void setLastModified(String lastModified) { - this.lastModified = lastModified; - } - public static EmailDomainSummary valueOf(ProfileEmailDomainEntity pem) { EmailDomainSummary form = new EmailDomainSummary(); @@ -40,14 +22,6 @@ public static EmailDomainSummary valueOf(ProfileEmailDomainEntity pem) { if(!PojoUtil.isEmpty(pem.getEmailDomain())) { form.setValue(pem.getEmailDomain()); } - - if (pem.getDateCreated() != null) { - form.setCreatedDate(org.orcid.pojo.ajaxForm.Date.valueOf(pem.getDateCreated()).toFuzzyDate().toString()); - } - - if (pem.getLastModified() !=null) { - form.setLastModified(org.orcid.pojo.ajaxForm.Date.valueOf(pem.getLastModified()).toFuzzyDate().toString()); - } } return form; } From d749f3a9ebd6173507dcfc98c6d1394b231605af Mon Sep 17 00:00:00 2001 From: github actions Date: Mon, 23 Sep 2024 21:36:15 +0000 Subject: [PATCH 02/18] v2.65.5 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 323b9a3c05e..2a2e9256221 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.65.5 - 2024-09-23 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.4...v2.65.5) + ## v2.65.4 - 2024-09-19 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.3...v2.65.4) From f4fa943e0329141c64b263c85f80bd3a6f9af449 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru Date: Thu, 26 Sep 2024 13:55:13 +0100 Subject: [PATCH 03/18] Removed all created date, last modified from email domain summary --- .../manager/impl/SummaryManagerImpl.java | 4 --- .../org/orcid/core/model/EmailDomain.java | 27 +++---------------- .../model/RecordSummaryMarshallingTest.java | 4 +-- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java index 3ecd48fb678..3fcd322e53d 100644 --- a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java @@ -524,8 +524,6 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci for (ProfileEmailDomainEntity ped : emailDomains) { ed = new EmailDomain(); ed.setValue(ped.getEmailDomain()); - ed.setCreatedDate(Date.valueOf(ped.getDateCreated())); - ed.setLastModified(Date.valueOf(ped.getLastModified())); edList.add(ed); } } @@ -533,8 +531,6 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci edList.stream().limit(3).forEach(t -> { EmailDomain ed = new EmailDomain(); ed.setValue(t.getValue()); - ed.setCreatedDate(t.getCreatedDate()); - ed.setLastModified(t.getLastModified()); emailDomainsTop3.add(ed); }); diff --git a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java index 779717dd90d..371b7948f04 100644 --- a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java +++ b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java @@ -15,16 +15,12 @@ import io.swagger.v3.oas.annotations.media.Schema; @XmlAccessorType(XmlAccessType.FIELD) -@XmlType(propOrder = { "value", "createdDate","lastModified"}) +@XmlType(propOrder = { "value"}) @XmlRootElement(name = "email-domain", namespace = "http://www.orcid.org/ns/summary") @Schema(description = "Email Domain") public class EmailDomain { @XmlElement(name = "value", namespace = "http://www.orcid.org/ns/summary") protected String value; - @XmlElement(name = "created-date", namespace = "http://www.orcid.org/ns/common") - protected Date createdDate; - @XmlElement(name = "last-modified-date", namespace = "http://www.orcid.org/ns/common") - protected Date lastModified; public String getValue() { return value; @@ -34,27 +30,10 @@ public void setValue(String value) { this.value = value; } - @XmlTransient - public Date getCreatedDate() { - return createdDate; - } - - public void setCreatedDate(Date createdDate) { - this.createdDate = createdDate; - } - - @XmlTransient - public Date getLastModified() { - return lastModified; - } - - public void setLastModified(Date lastModified) { - this.lastModified = lastModified; - } @Override public int hashCode() { - return Objects.hash(value, createdDate, lastModified); + return Objects.hash(value); } @Override @@ -66,6 +45,6 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; EmailDomain other = (EmailDomain) obj; - return Objects.equals(createdDate, other.createdDate) && Objects.equals(lastModified, other.lastModified) && Objects.equals(value, other.value); + return Objects.equals(value, other.value); } } diff --git a/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java b/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java index 72c5a541aae..d55d341835b 100644 --- a/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java +++ b/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java @@ -97,7 +97,7 @@ private RecordSummary getRecordSummary() { EmailDomains emailDomains = new EmailDomains(); emailDomains.setCount(4); emailDomains.setEmailDomains(new ArrayList()); - emailDomains.getEmailDomains().add(getEmailDomain("sometrusted.org", getEmailDomainCreatedDate(), getEmailDomainLastModified())); + emailDomains.getEmailDomains().add(getEmailDomain("sometrusted.org")); record.setEmailDomains(emailDomains); //Set education/qualifications @@ -163,7 +163,7 @@ private EducationQualification getEducationQualification(int putCode, String rol return eq; } - private EmailDomain getEmailDomain(String domainValue, Date created, Date modified) { + private EmailDomain getEmailDomain(String domainValue) { EmailDomain emailDomain = new EmailDomain(); emailDomain.setValue(domainValue); //emailDomain.setCreatedDate(created); From a9da77d247f4bf874d38efe6c856cd342a3c1e9c Mon Sep 17 00:00:00 2001 From: Camelia Dumitru Date: Fri, 27 Sep 2024 15:32:14 +0100 Subject: [PATCH 04/18] Added the code to store deleted items in panoply dw_deleted_items table The items will be processed by the script that runs daily and cleaned up after the item is deleted from the corresponding view --- .../impl/OrgAffiliationRelationDaoImpl.java | 66 ++++++++++++++++++- .../resources/orcid-persistence-context.xml | 16 +++++ orcid-utils/pom.xml | 20 ++++++ .../utils/panoply/PanoplyDeletedItem.java | 55 ++++++++++++++++ .../utils/panoply/PanoplyRedshiftClient.java | 36 ++++++++++ properties/development.properties | 12 +++- 6 files changed, 202 insertions(+), 3 deletions(-) create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index 51185a67132..de1c3a5bf1f 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,19 +3,32 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; +import org.orcid.utils.panoply.PanoplyDeletedItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - + + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); + + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0 ? true : false; + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setItemId(orgAffiliationRelationId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + return true; + } + return false; + } + + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { + //Store the deleted item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyDeletedItem(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store deleted affiliation in panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); } /** @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setClientSourceId(clientSourceId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } } @Override @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setOrcid(orcid); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } + } @Override diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index d57dde50df1..7857fb72b54 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,6 +457,22 @@ + + + + + + + + + + + + + + + + diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index d7b361ba22d..6ee5bc45646 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-context + + org.springframework + spring-jdbc + org.apache.commons commons-lang3 @@ -113,6 +117,22 @@ jsoup 1.15.4 + + + + com.amazon.redshift + redshift-jdbc42 + 2.1.0.30 + + + + + + com.zaxxer + HikariCP + 5.0.1 + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java new file mode 100644 index 00000000000..378b0185227 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -0,0 +1,55 @@ +package org.orcid.utils.panoply; + +public class PanoplyDeletedItem { + private Long id; + private String dwTable; + private Long itemId; + private String clientSourceId; + private String orcid; + + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; + public final String DW_WORK = "dw_work"; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getDwTable() { + return dwTable; + } + public void setDwTable(String dwTable) { + this.dwTable = dwTable; + } + public Long getItemId() { + return itemId; + } + public void setItemId(Long itemId) { + this.itemId = itemId; + } + public String getClientSourceId() { + return clientSourceId; + } + public void setClientSourceId(String clientSourceId) { + this.clientSourceId = clientSourceId; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + + @Override + public String toString() { + return "PanoplyDeletedItem{" + + "id=" + id + + ", dwTable='" + dwTable + '\'' + + ", itemId='" + itemId + '\'' + + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + + '}'; + } + +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java new file mode 100644 index 00000000000..3499593eb27 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -0,0 +1,36 @@ +package org.orcid.utils.panoply; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + + +import java.util.Date; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.orcid.utils.alerting.SlackManager; + +@Repository +public class PanoplyRedshiftClient { + + private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); + + @Autowired + @Qualifier("panoplyJdbcTemplate") + private JdbcTemplate panoplyJdbcTemplate; + + + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); + } + +} diff --git a/properties/development.properties b/properties/development.properties index 34eae804933..9186b22ceb8 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -#org.orcid.persistence.liquibase.enabled=false +org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx From 6fdf65e6418bf3835b239893ee6965c156b4758a Mon Sep 17 00:00:00 2001 From: leomendoza123 Date: Tue, 1 Oct 2024 21:24:34 -0600 Subject: [PATCH 05/18] Revert "Added the code to store deleted items in panoply dw_deleted_items table" --- .../impl/OrgAffiliationRelationDaoImpl.java | 66 +------------------ .../resources/orcid-persistence-context.xml | 16 ----- orcid-utils/pom.xml | 20 ------ .../utils/panoply/PanoplyDeletedItem.java | 55 ---------------- .../utils/panoply/PanoplyRedshiftClient.java | 36 ---------- properties/development.properties | 12 +--- 6 files changed, 3 insertions(+), 202 deletions(-) delete mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java delete mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index de1c3a5bf1f..51185a67132 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,32 +3,19 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.CompletableFuture; -import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; -import org.orcid.utils.panoply.PanoplyDeletedItem; -import org.orcid.utils.panoply.PanoplyRedshiftClient; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; -import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - - private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); - - @Value("${org.orcid.persistence.panoply.cleanup.production:false}") - private boolean enablePanoplyCleanupInProduction; - + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -42,11 +29,6 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0) { - if(enablePanoplyCleanupInProduction) { - PanoplyDeletedItem item = new PanoplyDeletedItem(); - item.setItemId(orgAffiliationRelationId); - item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); - storeDeletedItemInPanoply(item); - } - return true; - } - return false; - } - - private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { - //Store the deleted item in panoply Db without blocking - CompletableFuture.supplyAsync(() -> { - try { - panoplyClient.addPanoplyDeletedItem(item); - return true; - } catch (Exception e) { - LOG.error("Cannot store deleted affiliation in panoply ", e); - return false; - } - }).thenAccept(result -> { - if(! result) { - LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); - } - - }); + return query.executeUpdate() > 0 ? true : false; } /** @@ -241,14 +196,6 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); - if(query.executeUpdate() > 0) { - if(enablePanoplyCleanupInProduction) { - PanoplyDeletedItem item = new PanoplyDeletedItem(); - item.setClientSourceId(clientSourceId); - item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); - storeDeletedItemInPanoply(item); - } - } } @Override @@ -334,15 +281,6 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); - if(query.executeUpdate() > 0) { - if(enablePanoplyCleanupInProduction) { - PanoplyDeletedItem item = new PanoplyDeletedItem(); - item.setOrcid(orcid); - item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); - storeDeletedItemInPanoply(item); - } - } - } @Override diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 7857fb72b54..d57dde50df1 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,22 +457,6 @@ - - - - - - - - - - - - - - - - diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index 6ee5bc45646..d7b361ba22d 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,10 +43,6 @@ org.springframework spring-context - - org.springframework - spring-jdbc - org.apache.commons commons-lang3 @@ -117,22 +113,6 @@ jsoup 1.15.4 - - - - com.amazon.redshift - redshift-jdbc42 - 2.1.0.30 - - - - - - com.zaxxer - HikariCP - 5.0.1 - - diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java deleted file mode 100644 index 378b0185227..00000000000 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java +++ /dev/null @@ -1,55 +0,0 @@ -package org.orcid.utils.panoply; - -public class PanoplyDeletedItem { - private Long id; - private String dwTable; - private Long itemId; - private String clientSourceId; - private String orcid; - - public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; - public final String DW_WORK = "dw_work"; - - public Long getId() { - return id; - } - public void setId(Long id) { - this.id = id; - } - public String getDwTable() { - return dwTable; - } - public void setDwTable(String dwTable) { - this.dwTable = dwTable; - } - public Long getItemId() { - return itemId; - } - public void setItemId(Long itemId) { - this.itemId = itemId; - } - public String getClientSourceId() { - return clientSourceId; - } - public void setClientSourceId(String clientSourceId) { - this.clientSourceId = clientSourceId; - } - public String getOrcid() { - return orcid; - } - public void setOrcid(String orcid) { - this.orcid = orcid; - } - - @Override - public String toString() { - return "PanoplyDeletedItem{" + - "id=" + id + - ", dwTable='" + dwTable + '\'' + - ", itemId='" + itemId + '\'' + - ", clientSourceId='" + clientSourceId + '\'' + - ", orcid='" + orcid + '\'' + - '}'; - } - -} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java deleted file mode 100644 index 3499593eb27..00000000000 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.orcid.utils.panoply; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.stereotype.Repository; -import org.springframework.stereotype.Service; - - -import java.util.Date; - -import javax.annotation.Resource; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.orcid.utils.alerting.SlackManager; - -@Repository -public class PanoplyRedshiftClient { - - private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); - - @Autowired - @Qualifier("panoplyJdbcTemplate") - private JdbcTemplate panoplyJdbcTemplate; - - - public int addPanoplyDeletedItem(PanoplyDeletedItem item) { - LOG.debug("Adding deleted item to panoply DB: " + item.toString()); - String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; - return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); - } - -} diff --git a/properties/development.properties b/properties/development.properties index 9186b22ceb8..34eae804933 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,15 +258,5 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -org.orcid.persistence.liquibase.enabled=false +#org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr - -org.orcid.persistence.panoply.cleanup.production=false -# Panoply redshift database -org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver -org.orcid.core.utils.panoply.maxPoolSize=20 -org.orcid.core.utils.panoply.password=xxx -org.orcid.core.utils.panoply.idleConnectionTimeout=3600 -org.orcid.core.utils.panoply.connectionTimeout=36000 -org.orcid.core.utils.panoply.jdbcUrl=xxx -org.orcid.core.utils.panoply.username=xxx From ccf173061d0655799103a99189a771c204e3ddf5 Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 2 Oct 2024 03:47:10 +0000 Subject: [PATCH 06/18] v2.65.6 changelog update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a2e9256221..0e40966486f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v2.65.6 - 2024-10-02 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.5...v2.65.6) + +- [#7093](https://github.com/ORCID/ORCID-Source/pull/7093): Revert "Added the code to store deleted items in panoply dw_deleted_items table" +- [#7091](https://github.com/ORCID/ORCID-Source/pull/7091): Added the code to store deleted items in panoply dw_deleted_items table +- [#7090](https://github.com/ORCID/ORCID-Source/pull/7090): Removed all created date, last modified from email domain summary + ## v2.65.5 - 2024-09-23 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.4...v2.65.5) From 39aad67c37cbd236904fca61dc9d11e580f26cee Mon Sep 17 00:00:00 2001 From: Giles Westwood Date: Mon, 7 Oct 2024 15:32:01 +0100 Subject: [PATCH 07/18] feat: map check names to their workflow files to allow for searching --- .github/workflows/maven_test_publish.yml | 2 +- .github/workflows/maven_test_report.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven_test_publish.yml b/.github/workflows/maven_test_publish.yml index 995ed7b7d00..33a6c1132d6 100644 --- a/.github/workflows/maven_test_publish.yml +++ b/.github/workflows/maven_test_publish.yml @@ -165,7 +165,7 @@ jobs: uses: dorny/test-reporter@e9fa2f582c0ebbe2e263fd18fad744d52e0b0203 if: always() with: - name: "${{ inputs.project }}: Unit tests results" + name: "maven_test_publish_${{ inputs.project }}" path: ${{ inputs.project }}/target/surefire-reports/*.xml reporter: java-junit fail-on-error: true diff --git a/.github/workflows/maven_test_report.yml b/.github/workflows/maven_test_report.yml index 017fe9c7af9..e44b1f70c99 100644 --- a/.github/workflows/maven_test_report.yml +++ b/.github/workflows/maven_test_report.yml @@ -24,7 +24,7 @@ jobs: - uses: dorny/test-reporter@e9fa2f582c0ebbe2e263fd18fad744d52e0b0203 with: artifact: ${{ inputs.project }} - name: "${{ inputs.project }} Unit Tests" # Name of the check run which will be created + name: "maven_test_report_${{ inputs.project }}" # Name of the check run which will be created path: '*.xml' # Path to test results (inside artifact .zip) reporter: java-junit # Format of test results From 3b7017caeb2265472c6b40c805b5d47f2f86512a Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 9 Oct 2024 13:18:48 +0000 Subject: [PATCH 08/18] v2.66.0 changelog update --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e40966486f..95e0bc328ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v2.66.0 - 2024-10-09 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.6...v2.66.0) + +### Feature + +- map check names to their workflow files to allow for searching + ## v2.65.6 - 2024-10-02 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.5...v2.65.6) From bb4e1010bfdcadc1d5e261ac6e416a24a66a9427 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Wed, 9 Oct 2024 18:49:06 +0100 Subject: [PATCH 09/18] Added the date verified when verifying an email address (#7101) Co-authored-by: Angel Montenegro --- .../orcid/persistence/dao/impl/EmailDaoImpl.java | 2 +- .../persistence/jpa/entities/EmailEntity.java | 14 +++++++++++++- orcid-persistence/src/main/resources/db-master.xml | 1 + .../db/updates/add_date_verified_to_email.xml | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java index a0effb50ab8..3df5afeb243 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java @@ -123,7 +123,7 @@ public void addSourceToEmail(String sourceId, String email) { @Transactional @UpdateProfileLastModified public boolean verifyEmail(String email) { - Query query = entityManager.createNativeQuery("update email set is_verified = true, is_current=true, last_modified=now() where trim(lower(email)) = trim(lower(:email))"); + Query query = entityManager.createNativeQuery("update email set is_verified = true, is_current=true, last_modified=now(), date_verified=now() where trim(lower(email)) = trim(lower(:email))"); query.setParameter("email", email); return query.executeUpdate() > 0; } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java index d9dbd1b5a8e..6977ec86502 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java @@ -3,6 +3,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Date; import javax.persistence.Column; import javax.persistence.Entity; @@ -24,7 +25,8 @@ public class EmailEntity extends SourceAwareEntity implements OrcidAware private Boolean primary; private Boolean current; private Boolean verified; - private String visibility; + private String visibility; + private Date dateVerified; @Override @Id @@ -90,6 +92,15 @@ public String getVisibility() { public void setVisibility(String visibility) { this.visibility = visibility; } + + @Column(name = "date_verified") + public Date getDateVerified() { + return dateVerified; + } + + public void setDateVerified(Date dateVerified) { + this.dateVerified = dateVerified; + } public static Map mapByLowerCaseEmail(Collection emailEntities) { Map map = new HashMap<>(); @@ -109,5 +120,6 @@ public void clean() { visibility= null; verified = null; visibility = null; + dateVerified = null; } } \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index c1a531b9bb3..65dfe060bf5 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -404,4 +404,5 @@ + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml new file mode 100644 index 00000000000..2b1327cf016 --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file From 7ab48afeed59201ebbf7fcf0e5576cd2dd596c9d Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 9 Oct 2024 18:03:48 +0000 Subject: [PATCH 10/18] v2.66.1 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 95e0bc328ff..c427584637c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.66.1 - 2024-10-09 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.0...v2.66.1) + ## v2.66.0 - 2024-10-09 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.6...v2.66.0) From 68efbf6864b89f7911de272f3c24a8b4bade5728 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Thu, 10 Oct 2024 15:43:00 +0100 Subject: [PATCH 11/18] Changed the type from timestamp to timestamp with timezone for date verified (#7102) --- .../main/resources/db/updates/add_date_verified_to_email.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml index 2b1327cf016..474039fe4e2 100644 --- a/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml +++ b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml @@ -6,7 +6,7 @@ - + From 2be37019e365ec95b1e1e63310d41c903b00c0f7 Mon Sep 17 00:00:00 2001 From: github actions Date: Thu, 10 Oct 2024 14:59:33 +0000 Subject: [PATCH 12/18] v2.66.2 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c427584637c..de70c422689 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.66.2 - 2024-10-10 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.1...v2.66.2) + ## v2.66.1 - 2024-10-09 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.0...v2.66.1) From 6a075350a864a47ee12a1f540f246644ed1e9957 Mon Sep 17 00:00:00 2001 From: Angel Montenegro Date: Fri, 11 Oct 2024 12:08:54 -0600 Subject: [PATCH 13/18] Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table"" (#7100) * Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table"" * Fixed the failures in test suite --------- Co-authored-by: Camelia Dumitru --- .../impl/OrgAffiliationRelationDaoImpl.java | 66 ++++++++++++++++++- .../resources/orcid-persistence-context.xml | 21 +++++- .../properties/test-api-common.properties | 14 +++- .../properties/test-api-internal.properties | 14 +++- .../resources/properties/test-core.properties | 14 +++- .../resources/properties/test-db.properties | 12 ++++ .../properties/test-scheduler.properties | 12 ++++ .../resources/properties/test-ui.properties | 14 +++- orcid-utils/pom.xml | 20 ++++++ .../utils/panoply/PanoplyDeletedItem.java | 55 ++++++++++++++++ .../utils/panoply/PanoplyRedshiftClient.java | 36 ++++++++++ properties/development.properties | 12 +++- 12 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index 51185a67132..de1c3a5bf1f 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,19 +3,32 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; +import org.orcid.utils.panoply.PanoplyDeletedItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - + + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); + + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -29,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0 ? true : false; + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setItemId(orgAffiliationRelationId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + return true; + } + return false; + } + + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { + //Store the deleted item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyDeletedItem(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store deleted affiliation in panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); } /** @@ -196,6 +241,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setClientSourceId(clientSourceId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } } @Override @@ -281,6 +334,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); + if(query.executeUpdate() > 0) { + if(enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setOrcid(orcid); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } + } @Override diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index d57dde50df1..554d47b39a6 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,10 +457,29 @@ + + + + + + + + + + + + + + + + - + + + + diff --git a/orcid-test/src/main/resources/properties/test-api-common.properties b/orcid-test/src/main/resources/properties/test-api-common.properties index 537759cfde2..dc4889fc41d 100644 --- a/orcid-test/src/main/resources/properties/test-api-common.properties +++ b/orcid-test/src/main/resources/properties/test-api-common.properties @@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-api-internal.properties b/orcid-test/src/main/resources/properties/test-api-internal.properties index d85df6f3aba..284325b6265 100644 --- a/orcid-test/src/main/resources/properties/test-api-internal.properties +++ b/orcid-test/src/main/resources/properties/test-api-internal.properties @@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-core.properties b/orcid-test/src/main/resources/properties/test-core.properties index ab3c0577d4c..1dc1d3e9298 100644 --- a/orcid-test/src/main/resources/properties/test-core.properties +++ b/orcid-test/src/main/resources/properties/test-core.properties @@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-db.properties b/orcid-test/src/main/resources/properties/test-db.properties index 8f8e05d0d68..a2c679f182a 100644 --- a/orcid-test/src/main/resources/properties/test-db.properties +++ b/orcid-test/src/main/resources/properties/test-db.properties @@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true org.orcid.persistence.togglz.db.preferredTestQuery=select 1 org.orcid.persistence.togglz.db.numHelperThreads=5 org.orcid.persistence.togglz.cache.ttl=60000 + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties index 10d29640a51..f1d06636692 100644 --- a/orcid-test/src/main/resources/properties/test-scheduler.properties +++ b/orcid-test/src/main/resources/properties/test-scheduler.properties @@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-ui.properties b/orcid-test/src/main/resources/properties/test-ui.properties index 6784fbe3068..9741a3fc49b 100644 --- a/orcid-test/src/main/resources/properties/test-ui.properties +++ b/orcid-test/src/main/resources/properties/test-ui.properties @@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index d7b361ba22d..6ee5bc45646 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-context + + org.springframework + spring-jdbc + org.apache.commons commons-lang3 @@ -113,6 +117,22 @@ jsoup 1.15.4 + + + + com.amazon.redshift + redshift-jdbc42 + 2.1.0.30 + + + + + + com.zaxxer + HikariCP + 5.0.1 + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java new file mode 100644 index 00000000000..378b0185227 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -0,0 +1,55 @@ +package org.orcid.utils.panoply; + +public class PanoplyDeletedItem { + private Long id; + private String dwTable; + private Long itemId; + private String clientSourceId; + private String orcid; + + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; + public final String DW_WORK = "dw_work"; + + public Long getId() { + return id; + } + public void setId(Long id) { + this.id = id; + } + public String getDwTable() { + return dwTable; + } + public void setDwTable(String dwTable) { + this.dwTable = dwTable; + } + public Long getItemId() { + return itemId; + } + public void setItemId(Long itemId) { + this.itemId = itemId; + } + public String getClientSourceId() { + return clientSourceId; + } + public void setClientSourceId(String clientSourceId) { + this.clientSourceId = clientSourceId; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + + @Override + public String toString() { + return "PanoplyDeletedItem{" + + "id=" + id + + ", dwTable='" + dwTable + '\'' + + ", itemId='" + itemId + '\'' + + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + + '}'; + } + +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java new file mode 100644 index 00000000000..3499593eb27 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -0,0 +1,36 @@ +package org.orcid.utils.panoply; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + + +import java.util.Date; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.orcid.utils.alerting.SlackManager; + +@Repository +public class PanoplyRedshiftClient { + + private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); + + @Autowired + @Qualifier("panoplyJdbcTemplate") + private JdbcTemplate panoplyJdbcTemplate; + + + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); + } + +} diff --git a/properties/development.properties b/properties/development.properties index 34eae804933..9186b22ceb8 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -#org.orcid.persistence.liquibase.enabled=false +org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx From 36cb5b11e4d4eb4107e830b2664367cdb0c55d67 Mon Sep 17 00:00:00 2001 From: github actions Date: Fri, 11 Oct 2024 18:23:18 +0000 Subject: [PATCH 14/18] v2.66.3 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index de70c422689..a8472b1d1b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.66.3 - 2024-10-11 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.2...v2.66.3) + ## v2.66.2 - 2024-10-10 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.1...v2.66.2) From 64fa3ca3a77695a61a8183b39cdf1583d1c9ac34 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:46:30 +0100 Subject: [PATCH 15/18] Revert 7093 revert 7091 9339 delete orgaffiliationrelation rows in panoply for affiliations deleted in the registry (#7106) * Revert "Revert "Added the code to store deleted items in panoply dw_deleted_items table"" * Fixed the failures in test suite * Fixed formatting added the default props for panoply redshift bean * removed the unused imports --------- Co-authored-by: Angel Montenegro Co-authored-by: Angel Montenegro --- .../impl/OrgAffiliationRelationDaoImpl.java | 56 +++++++++---------- .../resources/orcid-persistence-context.xml | 8 +-- .../utils/panoply/PanoplyDeletedItem.java | 24 ++++---- .../utils/panoply/PanoplyRedshiftClient.java | 13 ++--- 4 files changed, 51 insertions(+), 50 deletions(-) diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index de1c3a5bf1f..bf849fbb405 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -13,7 +13,6 @@ import org.orcid.utils.panoply.PanoplyRedshiftClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.commons.logging.Log; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; @@ -23,12 +22,12 @@ import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { - + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); - + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") - private boolean enablePanoplyCleanupInProduction; - + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -42,11 +41,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0) { - if(enablePanoplyCleanupInProduction) { + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { PanoplyDeletedItem item = new PanoplyDeletedItem(); item.setItemId(orgAffiliationRelationId); item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); storeDeletedItemInPanoply(item); - } + } return true; } - return false; + return false; } - + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { - //Store the deleted item in panoply Db without blocking + // Store the deleted item in panoply Db without blocking CompletableFuture.supplyAsync(() -> { try { panoplyClient.addPanoplyDeletedItem(item); @@ -91,11 +90,11 @@ private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { LOG.error("Cannot store deleted affiliation in panoply ", e); return false; } - }).thenAccept(result -> { - if(! result) { - LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + }).thenAccept(result -> { + if (!result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result); } - + }); } @@ -241,13 +240,13 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); - if(query.executeUpdate() > 0) { - if(enablePanoplyCleanupInProduction) { + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { PanoplyDeletedItem item = new PanoplyDeletedItem(); item.setClientSourceId(clientSourceId); item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); storeDeletedItemInPanoply(item); - } + } } } @@ -334,15 +333,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); - if(query.executeUpdate() > 0) { - if(enablePanoplyCleanupInProduction) { + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { PanoplyDeletedItem item = new PanoplyDeletedItem(); item.setOrcid(orcid); item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); storeDeletedItemInPanoply(item); - } + } } - + } @Override @@ -447,8 +446,7 @@ public void revertUserOBODetails(List ids) { @SuppressWarnings("unchecked") @Override public List getIdsForUserOBORecords(int max) { - Query query = entityManager - .createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL"); + Query query = entityManager.createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL"); query.setMaxResults(max); return query.getResultList(); } @@ -461,14 +459,14 @@ public List getIdsOfOrgAffiliationRelationsReferencingClientProfiles query.setMaxResults(max); return query.getResultList(); } - + @Override @UpdateProfileLastModifiedAndIndexingStatus @Transactional public void persist(OrgAffiliationRelationEntity affiliation) { super.persist(affiliation); } - + @Override @UpdateProfileLastModifiedAndIndexingStatus @Transactional diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 554d47b39a6..629b3b163c1 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -460,13 +460,13 @@ - + - - - + + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java index 378b0185227..a39df453cde 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -6,50 +6,54 @@ public class PanoplyDeletedItem { private Long itemId; private String clientSourceId; private String orcid; - + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; public final String DW_WORK = "dw_work"; - + public Long getId() { return id; } + public void setId(Long id) { this.id = id; } + public String getDwTable() { return dwTable; } + public void setDwTable(String dwTable) { this.dwTable = dwTable; } + public Long getItemId() { return itemId; } + public void setItemId(Long itemId) { this.itemId = itemId; } + public String getClientSourceId() { return clientSourceId; } + public void setClientSourceId(String clientSourceId) { this.clientSourceId = clientSourceId; } + public String getOrcid() { return orcid; } + public void setOrcid(String orcid) { this.orcid = orcid; } - + @Override public String toString() { - return "PanoplyDeletedItem{" + - "id=" + id + - ", dwTable='" + dwTable + '\'' + - ", itemId='" + itemId + '\'' + - ", clientSourceId='" + clientSourceId + '\'' + - ", orcid='" + orcid + '\'' + - '}'; + return "PanoplyDeletedItem{" + "id=" + id + ", dwTable='" + dwTable + '\'' + ", itemId='" + itemId + '\'' + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + '}'; } } diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java index 3499593eb27..77c5db8ef16 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -8,7 +8,6 @@ import org.springframework.stereotype.Repository; import org.springframework.stereotype.Service; - import java.util.Date; import javax.annotation.Resource; @@ -21,16 +20,16 @@ public class PanoplyRedshiftClient { private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); - + @Autowired @Qualifier("panoplyJdbcTemplate") private JdbcTemplate panoplyJdbcTemplate; - - + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { - LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; - return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), + item.getDwTable()); } - + } From 14d1885acdb8d6cfa45fb81b4fea5d32b121e05b Mon Sep 17 00:00:00 2001 From: github actions Date: Fri, 11 Oct 2024 21:15:13 +0000 Subject: [PATCH 16/18] v2.66.4 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8472b1d1b9..b9dc9b81240 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.66.4 - 2024-10-11 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.3...v2.66.4) + ## v2.66.3 - 2024-10-11 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.2...v2.66.3) From 9aaeae2805f077f3ebcadb1a8e38c4911a90681d Mon Sep 17 00:00:00 2001 From: andrej romanov <50377758+auumgn@users.noreply.github.com> Date: Wed, 16 Oct 2024 00:00:01 +0300 Subject: [PATCH 17/18] Add verification date to email endpoint (#7104) * Added the date verified when verifying an email address * fix domain modification date value * formatting * test verification date logic * update orcid model version * set up verification date in the endpoint * fix verification date endpoint logic * configure mapper to process verification dates * add verification assertions --------- Co-authored-by: Camelia Dumitru Co-authored-by: Angel Montenegro --- .../adapter/v3/impl/MapperFacadeFactory.java | 1 + .../java/org/orcid/pojo/ajaxForm/Email.java | 18 ++++++++++++++++++ .../java/org/orcid/pojo/ajaxForm/Emails.java | 2 +- .../pojo/ajaxForm/ProfileEmailDomain.java | 5 +++-- .../adapter/v3/JpaJaxbEmailAdapterTest.java | 4 ++++ pom.xml | 2 +- 6 files changed, 28 insertions(+), 4 deletions(-) diff --git a/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java b/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java index f0c2c2f28b9..0e0cb01658b 100644 --- a/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java +++ b/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java @@ -500,6 +500,7 @@ public MapperFacade getEmailMapperFacade() { emailClassMap.field("primary", "primary"); emailClassMap.field("verified", "verified"); emailClassMap.fieldMap("visibility", "visibility").converter("visibilityConverter").add(); + emailClassMap.field("verificationDate.value", "dateVerified"); addV3DateFields(emailClassMap); registerSourceConverters(mapperFactory, emailClassMap); emailClassMap.register(); diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java index 7b65d9f3190..930174b27bf 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java @@ -32,6 +32,8 @@ public class Email implements ErrorsInterface { private Date lastModified; + private Date verificationDate; + private List errors = new ArrayList(); public static Email valueOf(org.orcid.jaxb.model.v3.release.record.Email e) { @@ -76,6 +78,14 @@ public static Email valueOf(org.orcid.jaxb.model.v3.release.record.Email e) { lastModifiedDate.setDay(String.valueOf(e.getLastModifiedDate().getValue().getDay())); email.setLastModified(lastModifiedDate); } + + if (e.getVerificationDate() != null) { + Date verificationDate = new Date(); + verificationDate.setYear(String.valueOf(e.getVerificationDate().getValue().getYear())); + verificationDate.setMonth(String.valueOf(e.getVerificationDate().getValue().getMonth())); + verificationDate.setDay(String.valueOf(e.getVerificationDate().getValue().getDay())); + email.setVerificationDate(verificationDate); + } } return email; } @@ -187,6 +197,14 @@ public void setLastModified(Date lastModified) { this.lastModified = lastModified; } + public Date getVerificationDate() { + return verificationDate; + } + + public void setVerificationDate(Date verificationDate) { + this.verificationDate = verificationDate; + } + public List getErrors() { return errors; } diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java index 1ea3ad73889..3d808766fc9 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java @@ -33,7 +33,7 @@ public static Emails valueOf(org.orcid.jaxb.model.v3.release.record.Emails e, Li public org.orcid.jaxb.model.v3.release.record.Emails toV3Emails() { org.orcid.jaxb.model.v3.release.record.Emails v3Emails = new org.orcid.jaxb.model.v3.release.record.Emails(); - if(emails != null && !emails.isEmpty()) { + if (emails != null && !emails.isEmpty()) { for(Email email : emails) { v3Emails.getEmails().add(email.toV3Email()); } diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java index 3ac0d064246..422442a130e 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java @@ -26,7 +26,7 @@ public static ProfileEmailDomain valueOf(ProfileEmailDomainEntity ed) { Date createdDate = new Date(); LocalDate date = ed.getDateCreated().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); createdDate.setYear(String.valueOf(date.getYear())); - createdDate.setMonth(String.valueOf(date.getMonth())); + createdDate.setMonth(String.valueOf(date.getMonthValue())); createdDate.setDay(String.valueOf(date.getDayOfMonth())); createdDate.setTimestamp(ed.getDateCreated().toInstant().toEpochMilli()); emailDomain.setCreatedDate(createdDate); @@ -36,10 +36,11 @@ public static ProfileEmailDomain valueOf(ProfileEmailDomainEntity ed) { Date lastModifiedDate = new Date(); LocalDate date = ed.getLastModified().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); lastModifiedDate.setYear(String.valueOf(date.getYear())); - lastModifiedDate.setMonth(String.valueOf(date.getMonth())); + lastModifiedDate.setMonth(String.valueOf(date.getMonthValue())); lastModifiedDate.setDay(String.valueOf(date.getDayOfMonth())); emailDomain.setLastModified(lastModifiedDate); } + } return emailDomain; } diff --git a/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java b/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java index 6fb86a0a529..65a70e2e6ac 100644 --- a/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java +++ b/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java @@ -101,6 +101,7 @@ public void testEmailToEmailEntity() throws JAXBException { assertNotNull(entity); assertNull(entity.getDateCreated()); assertNull(entity.getLastModified()); + assertNull(entity.getDateVerified()); assertEquals("user1@email.com", entity.getEmail()); assertEquals(org.orcid.jaxb.model.common_v2.Visibility.PUBLIC.name(), entity.getVisibility()); @@ -119,6 +120,8 @@ public void testEmailEntityToEmail() throws IllegalAccessException { assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getCreatedDate().getValue())); assertNotNull(email.getLastModifiedDate()); assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getLastModifiedDate().getValue())); + assertNotNull(email.getVerificationDate()); + assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getVerificationDate().getValue())); assertEquals("email@test.orcid.org", email.getEmail()); assertEquals(Visibility.PRIVATE, email.getVisibility()); @@ -159,6 +162,7 @@ private EmailEntity getEmailEntity() throws IllegalAccessException { Date date = DateUtils.convertToDate("2015-06-05T10:15:20"); EmailEntity result = new EmailEntity(); DateFieldsOnBaseEntityUtils.setDateFields(result, date); + result.setDateVerified(date); result.setEmail("email@test.orcid.org"); result.setCurrent(true); result.setPrimary(true); diff --git a/pom.xml b/pom.xml index 102707e2b93..c6024f0e181 100644 --- a/pom.xml +++ b/pom.xml @@ -918,7 +918,7 @@ the software. org.orcid orcid-model - 3.3.1 + 3.3.2 org.orcid From aa2c36a03a6ca8013c0b7b956e1ce106ff722d64 Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 15 Oct 2024 21:19:12 +0000 Subject: [PATCH 18/18] v2.66.5 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b9dc9b81240..a641ac3d527 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.66.5 - 2024-10-15 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.4...v2.66.5) + ## v2.66.4 - 2024-10-11 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.3...v2.66.4)