Skip to content

Commit

Permalink
Do a single query to extract non verified emails that have not been n…
Browse files Browse the repository at this point in the history 
…otified yet
  • Loading branch information
@amontenegro
amontenegro committed Nov 16, 2023
1 parent 7b4062c commit e375684
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 151 deletions.
4 changes: 1 addition & 3 deletions orcid-persistence/src/main/java/org/orcid/persistence/dao/ProfileDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,7 @@ public interface ProfileDao extends GenericDao<ProfileEntity, String> {

void updateLastModifiedDateAndIndexingStatusWithoutResult(String orcid, Date lastModified, IndexingStatus indexingStatus);

public List<Triple<String, Boolean, Date>> findEmailsUnverfiedDays(int daysUnverified, int maxResults);

public List<Triple<String, Boolean, String>> findEmailsUnverifiedDaysByEventType(int daysUnverified, int tooOldNumberOfDays);
public List<Triple<String, String, Boolean>> findEmailsUnverfiedDays(int daysUnverified, EmailEventType eventSent);

String retrieveOrcidType(String orcid);

Expand Down
69 changes: 29 additions & 40 deletions orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/ProfileDaoImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,23 +13,39 @@
import org.apache.commons.lang3.tuple.Triple;
import org.orcid.persistence.aop.UpdateProfileLastModifiedAndIndexingStatus;
import org.orcid.persistence.dao.ProfileDao;
import org.orcid.persistence.jpa.entities.EmailEventType;
import org.orcid.persistence.jpa.entities.IndexingStatus;
import org.orcid.persistence.jpa.entities.OrcidGrantedAuthority;
import org.orcid.persistence.jpa.entities.ProfileEntity;
import org.orcid.persistence.jpa.entities.ProfileEventEntity;
import org.orcid.persistence.jpa.entities.ProfileEventType;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.transaction.annotation.Transactional;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.transaction.annotation.Transactional;

public class ProfileDaoImpl extends GenericDaoImpl<ProfileEntity, String> implements ProfileDao {

private static final String PRIVATE_VISIBILITY = "PRIVATE";

private static final Logger LOGGER = LoggerFactory.getLogger(ProfileDaoImpl.class);

private static final String FIND_EMAILS_TO_SEND_VERIFICATION_REMINTER = "select orcid, email, is_verified \n"
+ "from email \n"
+ "where email in \n"
+ " (SELECT distinct(e.email) \n"
+ " FROM email e, email_event ev, profile p \n"
+ " WHERE e.is_verified = false \n"
+ " AND p.orcid = e.orcid \n"
+ " AND p.deprecated_date is null \n"
+ " AND p.profile_deactivation_date is null \n"
+ " AND p.account_expiry is null \n"
+ " AND e.date_created BETWEEN (DATE_TRUNC('day', now()) - CAST('{RANGE_START}' AS INTERVAL DAY)) AND (DATE_TRUNC('day', now()) - CAST('{RANGE_END}' AS INTERVAL DAY)) \n"
+ " AND NOT EXISTS \n"
+ " (SELECT email FROM email_event x WHERE x.email = e.email AND email_event_type IN ('{EVENT_SENT}')\n"
+ " )\n"
+ ") order by last_modified;";

@Value("${org.orcid.postgres.query.timeout:30000}")
private Integer queryTimeout;

Expand Down Expand Up @@ -136,47 +152,20 @@ public List<String> findUnclaimedNotIndexedAfterWaitPeriod(int waitPeriodDays, i

@SuppressWarnings("unchecked")
@Override
public List<Triple<String, Boolean, Date>> findEmailsUnverfiedDays(int daysUnverified, int maxResults) {
StringBuilder queryString = new StringBuilder("SELECT e.email, e.is_primary, e.date_created FROM email e ");
queryString.append("LEFT JOIN email_event ev ON e.email = ev.email ");
queryString.append("JOIN profile p on p.orcid = e.orcid and p.claimed = true ");
queryString.append("AND p.deprecated_date is null AND p.profile_deactivation_date is null AND p.account_expiry is null ");
queryString.append("where ev.email IS NULL " + "and e.is_verified = false ");
queryString.append("and e.date_created < (now() - CAST('").append(daysUnverified).append("' AS INTERVAL DAY)) ");
queryString.append("and (e.source_id = e.orcid OR e.source_id is null)");
queryString.append(" ORDER BY e.last_modified");

Query query = entityManager.createNativeQuery(queryString.toString());
query.setMaxResults(maxResults);
List<Object[]> dbInfo = query.getResultList();
List<Triple<String, Boolean, Date>> results = new ArrayList<Triple<String, Boolean, Date>>();
dbInfo.stream().forEach(element -> {
Triple<String, Boolean, Date> pair = Triple.of((String) element[0], (Boolean) element[1], (Date) element[2]);
results.add(pair);
});
return results;
}

@SuppressWarnings("unchecked")
@Override
public List<Triple<String, Boolean, String>> findEmailsUnverifiedDaysByEventType(int daysUnverified, int tooOldNumberOfDays) {
StringBuilder queryString = new StringBuilder("SELECT e.email, e.is_primary, ev.email_event_type FROM email e ");
queryString.append("LEFT JOIN email_event ev ON e.email = ev.email ");
queryString.append("JOIN profile p on p.orcid = e.orcid and p.claimed = true ");
queryString.append("AND p.deprecated_date is null AND p.profile_deactivation_date is null AND p.account_expiry is null ");
queryString.append("where e.is_verified = false ");
queryString.append("and e.date_created between (now() - CAST('").append(tooOldNumberOfDays).append("' AS INTERVAL DAY)) and (now() - CAST('")
.append(daysUnverified).append("' AS INTERVAL DAY)) ");
queryString.append("and e.date_created < (now() - CAST('").append(daysUnverified).append("' AS INTERVAL DAY)) ");
queryString.append("and (e.source_id = e.orcid OR e.source_id is null)");
queryString.append(" ORDER BY e.last_modified");
public List<Triple<String, String, Boolean>> findEmailsUnverfiedDays(int daysUnverified, EmailEventType eventSent) {
int rangeStart = daysUnverified;
int rangeEnd = daysUnverified - 1;

String queryString = FIND_EMAILS_TO_SEND_VERIFICATION_REMINTER.replace("{RANGE_START}", String.valueOf(rangeStart))
.replace("{RANGE_END}", String.valueOf(rangeEnd)).replace("{EVENT_SENT}", eventSent.name());
LOGGER.debug("Query to search unverified emails");
LOGGER.debug(queryString);
Query query = entityManager.createNativeQuery(queryString.toString());
List<Object[]> dbInfo = query.getResultList();
List<Triple<String, Boolean, String>> results = new ArrayList<Triple<String, Boolean, String>>();
List<Triple<String, String, Boolean>> results = new ArrayList<Triple<String, String, Boolean>>();
dbInfo.stream().forEach(element -> {
Triple<String, Boolean, String> pair = Triple.of((String) element[0], (Boolean) element[1], (String) element[2]);
results.add(pair);
Triple<String, String, Boolean> q = Triple.of((String) element[0], (String) element[1], (Boolean) element[2]);
results.add(q);
});
return results;
}
Expand Down
68 changes: 36 additions & 32 deletions orcid-persistence/src/test/java/org/orcid/persistence/dao/ProfileDaoTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

import java.util.Arrays;
import java.util.Calendar;
Expand All @@ -17,7 +16,6 @@
import javax.persistence.EntityManager;
import javax.persistence.Query;

import org.apache.commons.lang3.tuple.Triple;
import org.dbunit.dataset.DataSetException;
import org.joda.time.LocalDateTime;
import org.junit.AfterClass;
Expand All @@ -26,11 +24,11 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.orcid.persistence.jpa.entities.EmailEventEntity;
import org.orcid.persistence.jpa.entities.EmailEventType;
import org.orcid.persistence.jpa.entities.IndexingStatus;
import org.orcid.persistence.jpa.entities.ProfileEntity;
import org.orcid.persistence.jpa.entities.ProfileEventEntity;
import org.orcid.persistence.jpa.entities.ProfileEventType;
import org.orcid.persistence.util.Quadruple;
import org.orcid.test.DBUnitTest;
import org.orcid.test.OrcidJUnit4ClassRunner;
import org.springframework.test.annotation.Rollback;
Expand Down Expand Up @@ -393,51 +391,57 @@ public void findEmailsUnverfiedDaysTest() throws IllegalAccessException {
// Created today
assertEquals(1, insertEmailWithDateCreated("[email protected]", "bd22086b65b6259fe79f7844a6b6a369441733b9ef04eff762f3d640957b78f5", orcid, false, new Date()));

// Created a week ago
assertEquals(1, insertEmailWithDateCreated("[email protected]", "95770578974f683fb05c179a84f57c3fc7d4b260f8079fbc590080e51873bb67", orcid, false, LocalDateTime.now().minusDays(7).toDate()));
// Created 2 days ago
assertEquals(1, insertEmailWithDateCreated("[email protected]", "95770578974f683fb05c179a84f57c3fc7d4b260f8079fbc590080e51873bb67", orcid, false, LocalDateTime.now().minusDays(2).toDate()));

// Created 7 days ago
assertEquals(1, insertEmailWithDateCreated("[email protected]", "3cbebfc1de2500494fc95553c956e757cb1998149d366afb71888cdeb1550719", orcid, false, LocalDateTime.now().minusDays(7).toDate()));

// Created 15 days ago
assertEquals(1, insertEmailWithDateCreated("[email protected]", "3cbebfc1de2500494fc95553c956e757cb1998149d366afb71888cdeb1550719", orcid, false, LocalDateTime.now().minusDays(15).toDate()));
assertEquals(1, insertEmailWithDateCreated("[email protected]", "8fccc2e14b546b968033dee2efb1644623a31a7878b55ae8ad52951961ca3719", orcid, false, LocalDateTime.now().minusDays(15).toDate()));

// Created 2 days ago and verified
assertEquals(1, insertEmailWithDateCreated("[email protected]", "2f4812b9c675e9803a4bb616dd1bc241c8c9302ba5690a1ea9d48049a32e7c5f", orcid, true, LocalDateTime.now().minusDays(2).toDate()));

// Created 7 days ago and verified
assertEquals(1, insertEmailWithDateCreated("verified_1@test.orcid.org", "2f4812b9c675e9803a4bb616dd1bc241c8c9302ba5690a1ea9d48049a32e7c5f", orcid, true, LocalDateTime.now().minusDays(7).toDate()));
assertEquals(1, insertEmailWithDateCreated("verified_2@test.orcid.org", "896dea808bbf69bde1b177f27800e84d17763860bffde1dfd8ef200e79ff9971", orcid, true, LocalDateTime.now().minusDays(7).toDate()));

// Created 15 days ago and verified
assertEquals(1, insertEmailWithDateCreated("verified_2@test.orcid.org", "896dea808bbf69bde1b177f27800e84d17763860bffde1dfd8ef200e79ff9971", orcid, true, LocalDateTime.now().minusDays(15).toDate()));
assertEquals(1, insertEmailWithDateCreated("verified_3@test.orcid.org", "f98cce12446df199b852583ce677ecf9870ebe1b58df21bc4e7b01dea67daf01", orcid, true, LocalDateTime.now().minusDays(15).toDate()));

List<Triple<String, Boolean, Date>> results = profileDao.findEmailsUnverfiedDays(7, 100);
List<Quadruple<String, String, Boolean, String>> results = profileDao.findEmailsUnverfiedDays(2);
assertNotNull(results);
assertEquals(2, results.size());
assertEquals(1, results.size());
assertEquals(orcid, results.get(0).getFirst());
assertEquals("[email protected]", results.get(0).getSecond());

boolean found1 = false, found2 = false;
results = profileDao.findEmailsUnverfiedDays(7);
assertNotNull(results);
assertEquals(1, results.size());
assertEquals(orcid, results.get(0).getFirst());
assertEquals("[email protected]", results.get(0).getSecond());

for(Triple<String, Boolean, Date> element : results) {
assertNotNull(element.getRight());
if(element.getLeft().equals("[email protected]")) {
found1 = true;
} else if(element.getLeft().equals("[email protected]")) {
found2 = true;
} else {
fail("Unexpected email id: " + element.getRight());
}
}
results = profileDao.findEmailsUnverfiedDays(15);
assertNotNull(results);
assertEquals(1, results.size());
assertEquals(orcid, results.get(0).getFirst());
assertEquals("[email protected]", results.get(0).getSecond());

assertTrue(found1);
assertTrue(found2);
results = profileDao.findEmailsUnverfiedDays(6);
assertNotNull(results);
assertTrue(results.isEmpty());

// Put an email event on '[email protected]' and verify there is only one result
emailEventDao.persist(new EmailEventEntity("[email protected]", EmailEventType.VERIFY_EMAIL_7_DAYS_SENT));
results = profileDao.findEmailsUnverfiedDays(8);
assertNotNull(results);
assertTrue(results.isEmpty());

results = profileDao.findEmailsUnverfiedDays(7, 100);
results = profileDao.findEmailsUnverfiedDays(14);
assertNotNull(results);
assertEquals(1, results.size());
assertEquals("[email protected]", results.get(0).getLeft());
assertTrue(results.isEmpty());

// Put an email event on '[email protected]' and verify there is no result anymore
emailEventDao.persist(new EmailEventEntity("[email protected]", EmailEventType.VERIFY_EMAIL_TOO_OLD));
results = profileDao.findEmailsUnverfiedDays(7, 100);
results = profileDao.findEmailsUnverfiedDays(16);
assertNotNull(results);
assertTrue(results.isEmpty());
assertTrue(results.isEmpty());
}

private int insertEmailWithDateCreated(String email, String emailHash, String orcid, boolean isVerified, Date dateCreated) {
Expand Down
Loading

0 comments on commit e375684

Please sign in to comment.