From 49c3dd8cbe1b271947b290d085159416788c3807 Mon Sep 17 00:00:00 2001 From: amontenegro Date: Wed, 17 Jul 2024 13:32:10 -0600 Subject: [PATCH] Running locally --- .../orcid/core/groupIds/issn/IssnClient.java | 2 +- .../persistence/dao/GroupIdRecordDao.java | 3 +- .../dao/impl/GroupIdRecordDaoImpl.java | 18 ++++--- .../jpa/entities/GroupIdRecordEntity.java | 36 ++++++++++++- .../InvalidIssnGroupIdRecordEntity.java | 40 -------------- .../main/resources/META-INF/persistence.xml | 3 +- .../src/main/resources/db-master.xml | 1 + ...eason_and_sync_date_to_group_id_record.xml | 50 +++++++++++++++++ .../resources/orcid-persistence-context.xml | 4 -- .../manager/impl/IssnLoadManagerImpl.java | 1 - .../loader/source/issn/IssnLoadSource.java | 54 +++++++++---------- 11 files changed, 129 insertions(+), 83 deletions(-) delete mode 100644 orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java create mode 100644 orcid-persistence/src/main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml diff --git a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java index e042ebaaa54..8df8e935147 100644 --- a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java +++ b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java @@ -51,7 +51,7 @@ public IssnData getIssnData(String issn) throws TooManyRequestsException, Unexpe return null; } } catch (Exception e) { - LOG.warn("Error extracting issn data from json returned from issn portal "+ issn, e); + LOG.warn("Error extracting issn data from json returned from issn portal "+ issn); return null; } } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java index 7a7b0c22a1d..18e2d33d415 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java @@ -1,5 +1,6 @@ package org.orcid.persistence.dao; +import java.util.Date; import java.util.List; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; @@ -16,5 +17,5 @@ public interface GroupIdRecordDao extends GenericDao boolean duplicateExists(Long putCode, String groupId); - List getIssnRecordsSortedById(int batchSize, long initialId); + List getIssnRecordsSortedBySyncDate(int batchSize, Date syncTime); } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java index 7befb1feaa5..332011883f9 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java @@ -1,5 +1,6 @@ package org.orcid.persistence.dao.impl; +import java.util.Date; import java.util.List; import javax.persistence.Query; @@ -7,9 +8,13 @@ import org.orcid.persistence.dao.GroupIdRecordDao; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; +import org.springframework.beans.factory.annotation.Value; + +public class GroupIdRecordDaoImpl extends GenericDaoImpl implements GroupIdRecordDao { + + @Value("${org.orcid.persistence.groupIdRecord.retry.max:5}") + private int maxRetries; -public class GroupIdRecordDaoImpl extends GenericDaoImpl implements GroupIdRecordDao { - public GroupIdRecordDaoImpl() { super(GroupIdRecordEntity.class); } @@ -73,10 +78,11 @@ public boolean duplicateExists(Long putCode, String groupId) { } @Override - public List getIssnRecordsSortedById(int batchSize, long initialId) { - Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g LEFT OUTER JOIN invalid_issn_group_id_record p ON g.id = p.id where p.id IS NULL AND g.group_id like 'issn:%' and g.id > :initialId order by g.id", GroupIdRecordEntity.class); - query.setParameter("initialId", initialId); - query.setMaxResults(batchSize); + public List getIssnRecordsSortedBySyncDate(int batchSize, Date syncTime) { + Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g WHERE g.issn_loader_fail_count < :max AND g.group_id LIKE 'issn:%' AND (g.sync_date is null OR g.sync_date < :syncTime) ORDER BY g.sync_date", GroupIdRecordEntity.class); + query.setParameter("max", maxRetries); + query.setParameter("syncTime", syncTime); + query.setMaxResults(batchSize); return query.getResultList(); } } diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java index ec55352ecee..12720bd5fb8 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java @@ -7,6 +7,7 @@ import javax.persistence.Id; import javax.persistence.SequenceGenerator; import javax.persistence.Table; +import java.util.Date; /** * The persistent class for the group_id_record database table. @@ -26,7 +27,13 @@ public class GroupIdRecordEntity extends SourceAwareEntity implements Comp private String groupDescription; - private String groupType; + private String groupType; + + private Integer issnLoaderFailCount; + + private String failReason; + + private Date syncDate; @Id @GeneratedValue(strategy = GenerationType.AUTO, generator = "group_id_record_seq") @@ -75,6 +82,33 @@ public void setGroupType(String groupType) { this.groupType = groupType; } + @Column(name = "issn_loader_fail_count") + public Integer getIssnLoaderFailCount() { + return issnLoaderFailCount; + } + + public void setIssnLoaderFailCount(Integer issnLoaderFailCount) { + this.issnLoaderFailCount = issnLoaderFailCount; + } + + @Column(name = "fail_reason") + public String getFailReason() { + return failReason; + } + + public void setFailReason(String failReason) { + this.failReason = failReason; + } + + @Column(name = "sync_date") + public Date getSyncDate() { + return syncDate; + } + + public void setSyncDate(Date syncDate) { + this.syncDate = syncDate; + } + @Override public int compareTo(GroupIdRecordEntity o) { return 0; diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java deleted file mode 100644 index 3e69411d5c4..00000000000 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.orcid.persistence.jpa.entities; - -import java.io.Serializable; - -import javax.persistence.Column; -import javax.persistence.Entity; -import javax.persistence.Id; -import javax.persistence.Table; - -@Entity -@Table(name = "invalid_issn_group_id_record") -public class InvalidIssnGroupIdRecordEntity extends BaseEntity implements Serializable { - - private static final long serialVersionUID = 1L; - - private String notes; - - private Long id; - - @Override - @Id - @Column - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - @Column - public String getNotes() { - return notes; - } - - public void setNotes(String note) { - this.notes = note; - } - -} diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml index 2ef97de7a61..418d0cfa39d 100644 --- a/orcid-persistence/src/main/resources/META-INF/persistence.xml +++ b/orcid-persistence/src/main/resources/META-INF/persistence.xml @@ -89,8 +89,7 @@ org.orcid.persistence.jpa.entities.RejectedGroupingSuggestionEntity org.orcid.persistence.jpa.entities.ValidatedPublicProfileEntity org.orcid.persistence.jpa.entities.MemberOBOWhitelistedClientEntity - org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity - org.orcid.persistence.jpa.entities.OrgImportLogEntity + org.orcid.persistence.jpa.entities.OrgImportLogEntity org.orcid.statistics.jpa.entities.StatisticValuesEntity diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index b1fc81b796a..da9e14ed2b7 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -391,4 +391,5 @@ + diff --git a/orcid-persistence/src/main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml b/orcid-persistence/src/main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml new file mode 100644 index 00000000000..eca7c094ec7 --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml @@ -0,0 +1,50 @@ + + + + + + + + + ALTER TABLE group_id_record ADD COLUMN issn_loader_fail_count INTEGER default 0; + + + + + + + + + ALTER TABLE group_id_record ADD COLUMN fail_reason VARCHAR(50); + + + + + + + + + ALTER TABLE group_id_record ADD COLUMN sync_date timestamp; + + + + + + + + + create index group_id_record_issn_loader_fail_count_index on group_id_record(issn_loader_fail_count); + + + + + + + + + create index group_id_record_sync_date_index on group_id_record(sync_date); + + + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 285312d0a1b..cedd58617ac 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -271,10 +271,6 @@ - - - - diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java index de57046d74a..75c74f7d4ca 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java @@ -36,7 +36,6 @@ public void loadIssn() { LOGGER.info("Load ISSN for client : " + issnSource); issnLoadSource.loadIssn(issnSource); slackManager.sendAlert("Issn succesfully updated for client " + issnSource, slackChannel, slackUser); - } catch (Exception ex) { LOGGER.error("Error when running ISSN for client" + issnSource, ex); slackManager.sendAlert("Error when running ISSN for client " + issnSource, slackChannel, slackUser); diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java index e689116f764..a4e88d73fb9 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java @@ -2,6 +2,7 @@ import java.io.IOException; import java.net.URISyntaxException; +import java.util.Date; import java.util.List; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,11 +16,9 @@ import org.orcid.core.groupIds.issn.IssnData; import org.orcid.core.groupIds.issn.IssnValidator; import org.orcid.persistence.dao.ClientDetailsDao; -import org.orcid.persistence.dao.GenericDao; import org.orcid.persistence.dao.GroupIdRecordDao; import org.orcid.persistence.jpa.entities.ClientDetailsEntity; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; -import org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -35,7 +34,7 @@ public class IssnLoadSource { @Value("${org.orcid.scheduler.issnLoadSource.batchSize:5000}") private int batchSize; - @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:30000}") + @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:10000}") private int waitBetweenBatches; @Resource @@ -43,9 +42,6 @@ public class IssnLoadSource { @Resource(name="groupIdRecordDaoReadOnly") private GroupIdRecordDao groupIdRecordDaoReadOnly; - - @Resource - private GenericDao invalidIssnGroupIdRecordDao; @Resource(name="clientDetailsDaoReadOnly") private ClientDetailsDao clientDetailsDaoReadOnly; @@ -72,10 +68,10 @@ public void loadIssn(String issnSource) { } private void updateIssnGroupIdRecords() { - Long nextBatchStartId = 0L; + Date startTime = new Date(); // Get the first batch of issn's - LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId + " batch size: " + batchSize); - List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId); + LOG.info("Running the process to load ISSN info, starting time: " + startTime + " batch size: " + batchSize); + List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime); int batchCount = 0; int total = 0; while (!issnEntities.isEmpty()) { @@ -95,23 +91,23 @@ private void updateIssnGroupIdRecords() { } catch(TooManyRequestsException tmre) { //TODO: We are being rate limited, we have to pause LOG.warn("We are being rate limited by the issn portal"); - recordFailure(issnEntity.getId(), "RATE_LIMIT reached"); + recordFailure(issnEntity, "RATE_LIMIT reached"); } catch(UnexpectedResponseCodeException urce) { LOG.warn("Unexpected response code {} for issn {}", urce.getReceivedCode(), issn); - recordFailure(issnEntity.getId(), "Unexpected response code " + urce.getReceivedCode()); + recordFailure(issnEntity, "Unexpected response code " + urce.getReceivedCode()); } catch (IOException e) { LOG.warn("IOException for issn {}", issn); - recordFailure(issnEntity.getId(), "IOException"); + recordFailure(issnEntity, "IOException"); } catch (URISyntaxException e) { LOG.warn("URISyntaxException for issn {}", issn); - recordFailure(issnEntity.getId(), "URISyntaxException"); + recordFailure(issnEntity, "URISyntaxException"); } catch (InterruptedException e) { LOG.warn("InterruptedException for issn {}", issn); - recordFailure(issnEntity.getId(), "InterruptedException"); + recordFailure(issnEntity, "InterruptedException"); } } else { LOG.info("Issn for group record {} not valid: {}", issnEntity.getId(), issnEntity.getGroupId()); - recordFailure(issnEntity.getId(), "Invalid record"); + recordFailure(issnEntity, "Invalid record"); } try { // Lets sleep for 30 secs after processing one batch @@ -125,21 +121,22 @@ private void updateIssnGroupIdRecords() { // TODO Auto-generated catch block LOG.warn("Exception while pausing the issn loader", e); } - - if (issnEntity.getId() > nextBatchStartId) { - nextBatchStartId = issnEntity.getId(); - } } - LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId); - issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId); + LOG.info("Loading next batch of ISSN's"); + issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime); } + LOG.info("All ISSN records processed"); } - private void recordFailure(Long id, String notes) { - InvalidIssnGroupIdRecordEntity invalidIssn = new InvalidIssnGroupIdRecordEntity(); - invalidIssn.setId(id); - invalidIssn.setNotes(notes); - invalidIssnGroupIdRecordDao.persist(invalidIssn); + private void recordFailure(GroupIdRecordEntity issnEntity, String notes) { + issnEntity.setFailReason(notes); + issnEntity.setSyncDate(new Date()); + if(issnEntity.getIssnLoaderFailCount() == null) { + issnEntity.setIssnLoaderFailCount(1); + } else { + issnEntity.setIssnLoaderFailCount(issnEntity.getIssnLoaderFailCount() + 1); + } + groupIdRecordDao.merge(issnEntity); } private void updateIssnEntity(GroupIdRecordEntity issnEntity, IssnData issnData) { @@ -149,10 +146,13 @@ private void updateIssnEntity(GroupIdRecordEntity issnEntity, IssnData issnData) if(!StringUtils.equals(currentGroupName, updatedGroupName)) { issnEntity.setGroupName(updatedGroupName); issnEntity.setClientSourceId(orcidSource.getId()); + issnEntity.setSyncDate(new Date()); LOG.info("Updating Group id: " + issnEntity.getGroupId() + " | current group name: " + currentGroupName + " | group name to be updated: " + issnEntity.getGroupName()); groupIdRecordDao.merge(issnEntity); } else { - LOG.info("Group id: " + issnEntity.getGroupId() + " is up to date"); + issnEntity.setSyncDate(new Date()); + groupIdRecordDao.merge(issnEntity); + LOG.debug("Group id: " + issnEntity.getGroupId() + " is up to date"); } }