diff --git a/.github/workflows/maven_test_publish.yml b/.github/workflows/maven_test_publish.yml index 995ed7b7d00..33a6c1132d6 100644 --- a/.github/workflows/maven_test_publish.yml +++ b/.github/workflows/maven_test_publish.yml @@ -165,7 +165,7 @@ jobs: uses: dorny/test-reporter@e9fa2f582c0ebbe2e263fd18fad744d52e0b0203 if: always() with: - name: "${{ inputs.project }}: Unit tests results" + name: "maven_test_publish_${{ inputs.project }}" path: ${{ inputs.project }}/target/surefire-reports/*.xml reporter: java-junit fail-on-error: true diff --git a/.github/workflows/maven_test_report.yml b/.github/workflows/maven_test_report.yml index 017fe9c7af9..e44b1f70c99 100644 --- a/.github/workflows/maven_test_report.yml +++ b/.github/workflows/maven_test_report.yml @@ -24,7 +24,7 @@ jobs: - uses: dorny/test-reporter@e9fa2f582c0ebbe2e263fd18fad744d52e0b0203 with: artifact: ${{ inputs.project }} - name: "${{ inputs.project }} Unit Tests" # Name of the check run which will be created + name: "maven_test_report_${{ inputs.project }}" # Name of the check run which will be created path: '*.xml' # Path to test results (inside artifact .zip) reporter: java-junit # Format of test results diff --git a/CHANGELOG.md b/CHANGELOG.md index 7958c926896..6665393febc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,53 @@ +## v2.66.6 - 2024-10-28 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.5...v2.66.6) + +- [#7111](https://github.com/ORCID/ORCID-Source/pull/7111): update license reference + +## v2.66.5 - 2024-10-15 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.4...v2.66.5) + +## v2.66.4 - 2024-10-11 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.3...v2.66.4) + +## v2.66.3 - 2024-10-11 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.2...v2.66.3) + +## v2.66.2 - 2024-10-10 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.1...v2.66.2) + +## v2.66.1 - 2024-10-09 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.0...v2.66.1) + +## v2.66.0 - 2024-10-09 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.6...v2.66.0) + +### Feature + +- map check names to their workflow files to allow for searching + +## v2.65.6 - 2024-10-02 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.5...v2.65.6) + +- [#7093](https://github.com/ORCID/ORCID-Source/pull/7093): Revert "Added the code to store deleted items in panoply dw_deleted_items table" +- [#7091](https://github.com/ORCID/ORCID-Source/pull/7091): Added the code to store deleted items in panoply dw_deleted_items table +- [#7090](https://github.com/ORCID/ORCID-Source/pull/7090): Removed all created date, last modified from email domain summary + +## v2.65.5 - 2024-09-23 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.4...v2.65.5) + +## v2.65.4 - 2024-09-19 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.3...v2.65.4) + ## v2.65.3 - 2024-09-17 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.65.2...v2.65.3) diff --git a/PROJECTS.md b/PROJECTS.md index edeb832269a..9f4421d869c 100644 --- a/PROJECTS.md +++ b/PROJECTS.md @@ -12,7 +12,7 @@ Also see our active contributors: [CREDITS.md](https://github.com/ORCID/ORCID-Wo > > Copyright (c) 2013 ORCID, Inc. > Licensed under an MIT-Style License (MIT) -> https://github.com/ORCID/ORCID-Source/blob/master/LICENSE.md +> https://github.com/ORCID/ORCID-Source/blob/main/LICENSE > http://orcid.org/OpenSource/license > > This copyright and license information (including a link to the full license) shall be included in its entirety in all copies or substantial portion of the software. diff --git a/TESTAUTO.md b/TESTAUTO.md index 9233ff29480..bc9f89b445e 100644 --- a/TESTAUTO.md +++ b/TESTAUTO.md @@ -124,5 +124,5 @@ Mac: ```/Applications/Firefox.app/Contents/MacOS/firefox-bin``` [src/test/resources/test-client.properties](https://github.com/ORCID/ORCID-Source/blob/master/orcid-integration-test/src/test/resources/test-client.properties) # License -See [LICENSE.md](https://github.com/ORCID/ORCID-Source/blob/master/LICENSE.md) +See [LICENSE](https://github.com/ORCID/ORCID-Source/blob/main/LICENSE) diff --git a/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java b/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java index f0c2c2f28b9..0e0cb01658b 100644 --- a/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java +++ b/orcid-core/src/main/java/org/orcid/core/adapter/v3/impl/MapperFacadeFactory.java @@ -500,6 +500,7 @@ public MapperFacade getEmailMapperFacade() { emailClassMap.field("primary", "primary"); emailClassMap.field("verified", "verified"); emailClassMap.fieldMap("visibility", "visibility").converter("visibilityConverter").add(); + emailClassMap.field("verificationDate.value", "dateVerified"); addV3DateFields(emailClassMap); registerSourceConverters(mapperFactory, emailClassMap); emailClassMap.register(); diff --git a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java index 766f3f4f65a..3fcd322e53d 100644 --- a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/SummaryManagerImpl.java @@ -73,10 +73,11 @@ import org.orcid.pojo.summary.ExternalIdentifiersSummary; import org.orcid.pojo.summary.RecordSummaryPojo; import org.orcid.utils.DateUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; public class SummaryManagerImpl implements SummaryManager { - @Resource(name = "recordNameManagerReadOnlyV3") private RecordNameManagerReadOnly recordNameManagerReadOnly; @@ -285,8 +286,7 @@ public RecordSummaryPojo getRecordSummaryPojo(String orcid) { for (EmailDomain ed : recordSummary.getEmailDomains().getEmailDomains()) { EmailDomainSummary eds = new EmailDomainSummary(); eds.setValue(ed.getValue()); - eds.setCreatedDate(ed.getCreatedDate().toFuzzyDate().toString()); - eds.setLastModified(ed.getLastModified().toFuzzyDate().toString()); + emailDomains.add(eds); } } pojo.setEmailDomains(emailDomains); @@ -520,23 +520,17 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci emailDomains = profileEmailDomainManagerReadOnly.getPublicEmailDomains(orcid); List edList = new ArrayList(); if (emailDomains != null && !emailDomains.isEmpty()) { - EmailDomain ed = null; for (ProfileEmailDomainEntity ped : emailDomains) { ed = new EmailDomain(); ed.setValue(ped.getEmailDomain()); - ed.setCreatedDate(Date.valueOf(ped.getDateCreated())); - ed.setLastModified(Date.valueOf(ped.getLastModified())); edList.add(ed); } } - List emailDomainsTop3 = new ArrayList(); edList.stream().limit(3).forEach(t -> { EmailDomain ed = new EmailDomain(); ed.setValue(t.getValue()); - ed.setCreatedDate(t.getCreatedDate()); - ed.setLastModified(t.getLastModified()); emailDomainsTop3.add(ed); }); @@ -544,7 +538,9 @@ public void generateEmailDomainsSummary(RecordSummary recordSummary, String orci eds.setCount(edList.size()); if (!emailDomainsTop3.isEmpty()) { eds.setEmailDomains(emailDomainsTop3); + } + recordSummary.setEmailDomains(eds); } } diff --git a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java index a31928c6841..371b7948f04 100644 --- a/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java +++ b/orcid-core/src/main/java/org/orcid/core/model/EmailDomain.java @@ -6,6 +6,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlElement; import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; import org.orcid.jaxb.model.v3.release.common.FuzzyDate; @@ -14,16 +15,12 @@ import io.swagger.v3.oas.annotations.media.Schema; @XmlAccessorType(XmlAccessType.FIELD) -@XmlType(propOrder = { "value","createdDate", "lastModified"}) -@XmlRootElement(name = "education-qualification", namespace = "http://www.orcid.org/ns/summary") -@Schema(description = "Education Qualification") +@XmlType(propOrder = { "value"}) +@XmlRootElement(name = "email-domain", namespace = "http://www.orcid.org/ns/summary") +@Schema(description = "Email Domain") public class EmailDomain { @XmlElement(name = "value", namespace = "http://www.orcid.org/ns/summary") protected String value; - @XmlElement(name = "created-date", namespace = "http://www.orcid.org/ns/common") - protected Date createdDate; - @XmlElement(name = "last-modified-date", namespace = "http://www.orcid.org/ns/common") - protected Date lastModified; public String getValue() { return value; @@ -32,26 +29,11 @@ public String getValue() { public void setValue(String value) { this.value = value; } - - public Date getCreatedDate() { - return createdDate; - } - - public void setCreatedDate(Date createdDate) { - this.createdDate = createdDate; - } - - public Date getLastModified() { - return lastModified; - } - - public void setLastModified(Date lastModified) { - this.lastModified = lastModified; - } + @Override public int hashCode() { - return Objects.hash(value, createdDate, lastModified); + return Objects.hash(value); } @Override @@ -63,6 +45,6 @@ public boolean equals(Object obj) { if (getClass() != obj.getClass()) return false; EmailDomain other = (EmailDomain) obj; - return Objects.equals(createdDate, other.createdDate) && Objects.equals(lastModified, other.lastModified) && Objects.equals(value, other.value); + return Objects.equals(value, other.value); } } diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java index 7b65d9f3190..930174b27bf 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Email.java @@ -32,6 +32,8 @@ public class Email implements ErrorsInterface { private Date lastModified; + private Date verificationDate; + private List errors = new ArrayList(); public static Email valueOf(org.orcid.jaxb.model.v3.release.record.Email e) { @@ -76,6 +78,14 @@ public static Email valueOf(org.orcid.jaxb.model.v3.release.record.Email e) { lastModifiedDate.setDay(String.valueOf(e.getLastModifiedDate().getValue().getDay())); email.setLastModified(lastModifiedDate); } + + if (e.getVerificationDate() != null) { + Date verificationDate = new Date(); + verificationDate.setYear(String.valueOf(e.getVerificationDate().getValue().getYear())); + verificationDate.setMonth(String.valueOf(e.getVerificationDate().getValue().getMonth())); + verificationDate.setDay(String.valueOf(e.getVerificationDate().getValue().getDay())); + email.setVerificationDate(verificationDate); + } } return email; } @@ -187,6 +197,14 @@ public void setLastModified(Date lastModified) { this.lastModified = lastModified; } + public Date getVerificationDate() { + return verificationDate; + } + + public void setVerificationDate(Date verificationDate) { + this.verificationDate = verificationDate; + } + public List getErrors() { return errors; } diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java index 1ea3ad73889..3d808766fc9 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/Emails.java @@ -33,7 +33,7 @@ public static Emails valueOf(org.orcid.jaxb.model.v3.release.record.Emails e, Li public org.orcid.jaxb.model.v3.release.record.Emails toV3Emails() { org.orcid.jaxb.model.v3.release.record.Emails v3Emails = new org.orcid.jaxb.model.v3.release.record.Emails(); - if(emails != null && !emails.isEmpty()) { + if (emails != null && !emails.isEmpty()) { for(Email email : emails) { v3Emails.getEmails().add(email.toV3Email()); } diff --git a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java index 3ac0d064246..422442a130e 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java +++ b/orcid-core/src/main/java/org/orcid/pojo/ajaxForm/ProfileEmailDomain.java @@ -26,7 +26,7 @@ public static ProfileEmailDomain valueOf(ProfileEmailDomainEntity ed) { Date createdDate = new Date(); LocalDate date = ed.getDateCreated().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); createdDate.setYear(String.valueOf(date.getYear())); - createdDate.setMonth(String.valueOf(date.getMonth())); + createdDate.setMonth(String.valueOf(date.getMonthValue())); createdDate.setDay(String.valueOf(date.getDayOfMonth())); createdDate.setTimestamp(ed.getDateCreated().toInstant().toEpochMilli()); emailDomain.setCreatedDate(createdDate); @@ -36,10 +36,11 @@ public static ProfileEmailDomain valueOf(ProfileEmailDomainEntity ed) { Date lastModifiedDate = new Date(); LocalDate date = ed.getLastModified().toInstant().atZone(ZoneId.systemDefault()).toLocalDate(); lastModifiedDate.setYear(String.valueOf(date.getYear())); - lastModifiedDate.setMonth(String.valueOf(date.getMonth())); + lastModifiedDate.setMonth(String.valueOf(date.getMonthValue())); lastModifiedDate.setDay(String.valueOf(date.getDayOfMonth())); emailDomain.setLastModified(lastModifiedDate); } + } return emailDomain; } diff --git a/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java b/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java index 4513d3db408..a8e6c1ba0a8 100644 --- a/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java +++ b/orcid-core/src/main/java/org/orcid/pojo/summary/EmailDomainSummary.java @@ -5,8 +5,6 @@ public class EmailDomainSummary { private String value; - private String createdDate; - private String lastModified; public String getValue() { return value; @@ -16,22 +14,6 @@ public void setValue(String value) { this.value = value; } - public String getCreatedDate() { - return createdDate; - } - - public void setCreatedDate(String createdDate) { - this.createdDate = createdDate; - } - - public String getLastModified() { - return lastModified; - } - - public void setLastModified(String lastModified) { - this.lastModified = lastModified; - } - public static EmailDomainSummary valueOf(ProfileEmailDomainEntity pem) { EmailDomainSummary form = new EmailDomainSummary(); @@ -40,14 +22,6 @@ public static EmailDomainSummary valueOf(ProfileEmailDomainEntity pem) { if(!PojoUtil.isEmpty(pem.getEmailDomain())) { form.setValue(pem.getEmailDomain()); } - - if (pem.getDateCreated() != null) { - form.setCreatedDate(org.orcid.pojo.ajaxForm.Date.valueOf(pem.getDateCreated()).toFuzzyDate().toString()); - } - - if (pem.getLastModified() !=null) { - form.setLastModified(org.orcid.pojo.ajaxForm.Date.valueOf(pem.getLastModified()).toFuzzyDate().toString()); - } } return form; } diff --git a/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java b/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java index 6fb86a0a529..65a70e2e6ac 100644 --- a/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java +++ b/orcid-core/src/test/java/org/orcid/core/adapter/v3/JpaJaxbEmailAdapterTest.java @@ -101,6 +101,7 @@ public void testEmailToEmailEntity() throws JAXBException { assertNotNull(entity); assertNull(entity.getDateCreated()); assertNull(entity.getLastModified()); + assertNull(entity.getDateVerified()); assertEquals("user1@email.com", entity.getEmail()); assertEquals(org.orcid.jaxb.model.common_v2.Visibility.PUBLIC.name(), entity.getVisibility()); @@ -119,6 +120,8 @@ public void testEmailEntityToEmail() throws IllegalAccessException { assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getCreatedDate().getValue())); assertNotNull(email.getLastModifiedDate()); assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getLastModifiedDate().getValue())); + assertNotNull(email.getVerificationDate()); + assertEquals(DateUtils.convertToDate("2015-06-05T10:15:20"), DateUtils.convertToDate(email.getVerificationDate().getValue())); assertEquals("email@test.orcid.org", email.getEmail()); assertEquals(Visibility.PRIVATE, email.getVisibility()); @@ -159,6 +162,7 @@ private EmailEntity getEmailEntity() throws IllegalAccessException { Date date = DateUtils.convertToDate("2015-06-05T10:15:20"); EmailEntity result = new EmailEntity(); DateFieldsOnBaseEntityUtils.setDateFields(result, date); + result.setDateVerified(date); result.setEmail("email@test.orcid.org"); result.setCurrent(true); result.setPrimary(true); diff --git a/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java b/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java index 72c5a541aae..d55d341835b 100644 --- a/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java +++ b/orcid-core/src/test/java/org/orcid/core/model/RecordSummaryMarshallingTest.java @@ -97,7 +97,7 @@ private RecordSummary getRecordSummary() { EmailDomains emailDomains = new EmailDomains(); emailDomains.setCount(4); emailDomains.setEmailDomains(new ArrayList()); - emailDomains.getEmailDomains().add(getEmailDomain("sometrusted.org", getEmailDomainCreatedDate(), getEmailDomainLastModified())); + emailDomains.getEmailDomains().add(getEmailDomain("sometrusted.org")); record.setEmailDomains(emailDomains); //Set education/qualifications @@ -163,7 +163,7 @@ private EducationQualification getEducationQualification(int putCode, String rol return eq; } - private EmailDomain getEmailDomain(String domainValue, Date created, Date modified) { + private EmailDomain getEmailDomain(String domainValue) { EmailDomain emailDomain = new EmailDomain(); emailDomain.setValue(domainValue); //emailDomain.setCreatedDate(created); diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java index a0effb50ab8..3df5afeb243 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EmailDaoImpl.java @@ -123,7 +123,7 @@ public void addSourceToEmail(String sourceId, String email) { @Transactional @UpdateProfileLastModified public boolean verifyEmail(String email) { - Query query = entityManager.createNativeQuery("update email set is_verified = true, is_current=true, last_modified=now() where trim(lower(email)) = trim(lower(:email))"); + Query query = entityManager.createNativeQuery("update email set is_verified = true, is_current=true, last_modified=now(), date_verified=now() where trim(lower(email)) = trim(lower(:email))"); query.setParameter("email", email); return query.executeUpdate() > 0; } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java index 51185a67132..bf849fbb405 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/OrgAffiliationRelationDaoImpl.java @@ -3,19 +3,31 @@ import java.math.BigInteger; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.persistence.Query; import javax.persistence.TypedQuery; +import org.orcid.utils.panoply.PanoplyDeletedItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.orcid.persistence.aop.UpdateProfileLastModified; import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus; import org.orcid.persistence.dao.OrgAffiliationRelationDao; import org.orcid.persistence.jpa.entities.OrgAffiliationRelationEntity; +import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.transaction.annotation.Transactional; public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl implements OrgAffiliationRelationDao { + private static final Logger LOG = LoggerFactory.getLogger(OrgAffiliationRelationDaoImpl.class); + + @Value("${org.orcid.persistence.panoply.cleanup.production:false}") + private boolean enablePanoplyCleanupInProduction; + private static final String AFFILIATION_TYPE_DISTINCTION = "DISTINCTION"; private static final String AFFILIATION_TYPE_EDUCATION = "EDUCATION"; @@ -30,6 +42,11 @@ public class OrgAffiliationRelationDaoImpl extends GenericDaoImpl 0 ? true : false; + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setItemId(orgAffiliationRelationId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + return true; + } + return false; + } + + private void storeDeletedItemInPanoply(PanoplyDeletedItem item) { + // Store the deleted item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyDeletedItem(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store deleted affiliation in panoply ", e); + return false; + } + }).thenAccept(result -> { + if (!result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result); + } + + }); } /** @@ -196,6 +240,14 @@ public void removeOrgAffiliationByClientSourceId(String clientSourceId) { Query query = entityManager.createNativeQuery("DELETE FROM org_affiliation_relation WHERE client_source_id=:clientSourceId"); query.setParameter("clientSourceId", clientSourceId); query.executeUpdate(); + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setClientSourceId(clientSourceId); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } } @Override @@ -281,6 +333,15 @@ public void removeAllAffiliations(String orcid) { Query query = entityManager.createQuery("delete from OrgAffiliationRelationEntity where orcid = :orcid"); query.setParameter("orcid", orcid); query.executeUpdate(); + if (query.executeUpdate() > 0) { + if (enablePanoplyCleanupInProduction) { + PanoplyDeletedItem item = new PanoplyDeletedItem(); + item.setOrcid(orcid); + item.setDwTable(DW_PANOPLY_AFFILIATION_TABLE); + storeDeletedItemInPanoply(item); + } + } + } @Override @@ -385,8 +446,7 @@ public void revertUserOBODetails(List ids) { @SuppressWarnings("unchecked") @Override public List getIdsForUserOBORecords(int max) { - Query query = entityManager - .createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL"); + Query query = entityManager.createNativeQuery("SELECT id FROM org_affiliation_relation WHERE assertion_origin_source_id IS NOT NULL"); query.setMaxResults(max); return query.getResultList(); } @@ -399,14 +459,14 @@ public List getIdsOfOrgAffiliationRelationsReferencingClientProfiles query.setMaxResults(max); return query.getResultList(); } - + @Override @UpdateProfileLastModifiedAndIndexingStatus @Transactional public void persist(OrgAffiliationRelationEntity affiliation) { super.persist(affiliation); } - + @Override @UpdateProfileLastModifiedAndIndexingStatus @Transactional diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java index d9dbd1b5a8e..6977ec86502 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EmailEntity.java @@ -3,6 +3,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; +import java.util.Date; import javax.persistence.Column; import javax.persistence.Entity; @@ -24,7 +25,8 @@ public class EmailEntity extends SourceAwareEntity implements OrcidAware private Boolean primary; private Boolean current; private Boolean verified; - private String visibility; + private String visibility; + private Date dateVerified; @Override @Id @@ -90,6 +92,15 @@ public String getVisibility() { public void setVisibility(String visibility) { this.visibility = visibility; } + + @Column(name = "date_verified") + public Date getDateVerified() { + return dateVerified; + } + + public void setDateVerified(Date dateVerified) { + this.dateVerified = dateVerified; + } public static Map mapByLowerCaseEmail(Collection emailEntities) { Map map = new HashMap<>(); @@ -109,5 +120,6 @@ public void clean() { visibility= null; verified = null; visibility = null; + dateVerified = null; } } \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index c1a531b9bb3..65dfe060bf5 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -404,4 +404,5 @@ + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml new file mode 100644 index 00000000000..474039fe4e2 --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/add_date_verified_to_email.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index d57dde50df1..629b3b163c1 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -457,10 +457,29 @@ + + + + + + + + + + + + + + + + - + + + + diff --git a/orcid-test/src/main/resources/properties/test-api-common.properties b/orcid-test/src/main/resources/properties/test-api-common.properties index 537759cfde2..dc4889fc41d 100644 --- a/orcid-test/src/main/resources/properties/test-api-common.properties +++ b/orcid-test/src/main/resources/properties/test-api-common.properties @@ -9,4 +9,16 @@ org.orcid.swagger.authendpoint=https://localhost:8443/orcid-web/oauth/authorize # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-api-internal.properties b/orcid-test/src/main/resources/properties/test-api-internal.properties index d85df6f3aba..284325b6265 100644 --- a/orcid-test/src/main/resources/properties/test-api-internal.properties +++ b/orcid-test/src/main/resources/properties/test-api-internal.properties @@ -9,4 +9,16 @@ org.orcid.persistence.internal_api.db.readonly.maxPoolSize=3 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-core.properties b/orcid-test/src/main/resources/properties/test-core.properties index ab3c0577d4c..1dc1d3e9298 100644 --- a/orcid-test/src/main/resources/properties/test-core.properties +++ b/orcid-test/src/main/resources/properties/test-core.properties @@ -76,4 +76,16 @@ org.orcid.core.orgsToGroup.query=select a.* from org_disambiguated a full outer # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-test/src/main/resources/properties/test-db.properties b/orcid-test/src/main/resources/properties/test-db.properties index 8f8e05d0d68..a2c679f182a 100644 --- a/orcid-test/src/main/resources/properties/test-db.properties +++ b/orcid-test/src/main/resources/properties/test-db.properties @@ -46,3 +46,15 @@ org.orcid.persistence.togglz.db.testConnectionOnCheckin=true org.orcid.persistence.togglz.db.preferredTestQuery=select 1 org.orcid.persistence.togglz.db.numHelperThreads=5 org.orcid.persistence.togglz.cache.ttl=60000 + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties index 10d29640a51..f1d06636692 100644 --- a/orcid-test/src/main/resources/properties/test-scheduler.properties +++ b/orcid-test/src/main/resources/properties/test-scheduler.properties @@ -42,3 +42,15 @@ org.orcid.core.orgs.ror.localZipPath=/tmp/ror/ror.zip org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx diff --git a/orcid-test/src/main/resources/properties/test-ui.properties b/orcid-test/src/main/resources/properties/test-ui.properties index 6784fbe3068..9741a3fc49b 100644 --- a/orcid-test/src/main/resources/properties/test-ui.properties +++ b/orcid-test/src/main/resources/properties/test-ui.properties @@ -28,4 +28,16 @@ org.orcid.core.profile.lockout.threshhold=10 # Redis org.orcid.core.utils.cache.redis.host=xxx.xxx.com org.orcid.core.utils.cache.redis.port=6379 -org.orcid.core.utils.cache.redis.password=XXXX \ No newline at end of file +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.core.utils.cache.redis.password=XXXX + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-utils/pom.xml b/orcid-utils/pom.xml index d7b361ba22d..6ee5bc45646 100644 --- a/orcid-utils/pom.xml +++ b/orcid-utils/pom.xml @@ -43,6 +43,10 @@ org.springframework spring-context + + org.springframework + spring-jdbc + org.apache.commons commons-lang3 @@ -113,6 +117,22 @@ jsoup 1.15.4 + + + + com.amazon.redshift + redshift-jdbc42 + 2.1.0.30 + + + + + + com.zaxxer + HikariCP + 5.0.1 + + diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java new file mode 100644 index 00000000000..a39df453cde --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyDeletedItem.java @@ -0,0 +1,59 @@ +package org.orcid.utils.panoply; + +public class PanoplyDeletedItem { + private Long id; + private String dwTable; + private Long itemId; + private String clientSourceId; + private String orcid; + + public final String DW_ORG_AFFILIATION_RELATION = "dw_org_affiliation_relation"; + public final String DW_WORK = "dw_work"; + + public Long getId() { + return id; + } + + public void setId(Long id) { + this.id = id; + } + + public String getDwTable() { + return dwTable; + } + + public void setDwTable(String dwTable) { + this.dwTable = dwTable; + } + + public Long getItemId() { + return itemId; + } + + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + public String getClientSourceId() { + return clientSourceId; + } + + public void setClientSourceId(String clientSourceId) { + this.clientSourceId = clientSourceId; + } + + public String getOrcid() { + return orcid; + } + + public void setOrcid(String orcid) { + this.orcid = orcid; + } + + @Override + public String toString() { + return "PanoplyDeletedItem{" + "id=" + id + ", dwTable='" + dwTable + '\'' + ", itemId='" + itemId + '\'' + ", clientSourceId='" + clientSourceId + '\'' + + ", orcid='" + orcid + '\'' + '}'; + } + +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java new file mode 100644 index 00000000000..77c5db8ef16 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -0,0 +1,35 @@ +package org.orcid.utils.panoply; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + +import java.util.Date; + +import javax.annotation.Resource; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.orcid.utils.alerting.SlackManager; + +@Repository +public class PanoplyRedshiftClient { + + private static final Logger LOG = LoggerFactory.getLogger(PanoplyRedshiftClient.class); + + @Autowired + @Qualifier("panoplyJdbcTemplate") + private JdbcTemplate panoplyJdbcTemplate; + + public int addPanoplyDeletedItem(PanoplyDeletedItem item) { + LOG.debug("Adding deleted item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_deleted_items (item_id, orcid, client_source_id, date_deleted, dw_table) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), + item.getDwTable()); + } + +} diff --git a/pom.xml b/pom.xml index 102707e2b93..c6024f0e181 100644 --- a/pom.xml +++ b/pom.xml @@ -918,7 +918,7 @@ the software. org.orcid orcid-model - 3.3.1 + 3.3.2 org.orcid diff --git a/properties/development.properties b/properties/development.properties index 34eae804933..9186b22ceb8 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,5 +258,15 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -#org.orcid.persistence.liquibase.enabled=false +org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx