Running locally

ORCID · Jul 17, 2024 · 49c3dd8 · 49c3dd8
1 parent 2f8657f
commit 49c3dd8
Show file tree

Hide file tree

Showing 11 changed files with 129 additions and 83 deletions.
diff --git a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java
@@ -51,7 +51,7 @@ public IssnData getIssnData(String issn) throws TooManyRequestsException, Unexpe
                 return null;
             }
         } catch (Exception e) {
-            LOG.warn("Error extracting issn data from json returned from issn portal "+ issn, e);
+            LOG.warn("Error extracting issn data from json returned from issn portal "+ issn);
             return null;
         }
     }

diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/GroupIdRecordDao.java
@@ -1,5 +1,6 @@
 package org.orcid.persistence.dao;
 
+import java.util.Date;
 import java.util.List;
 
 import org.orcid.persistence.jpa.entities.GroupIdRecordEntity;
@@ -16,5 +17,5 @@ public interface GroupIdRecordDao extends GenericDao<GroupIdRecordEntity, Long>
 
     boolean duplicateExists(Long putCode, String groupId);
 
-    List<GroupIdRecordEntity> getIssnRecordsSortedById(int batchSize, long initialId);
+    List<GroupIdRecordEntity> getIssnRecordsSortedBySyncDate(int batchSize, Date syncTime);
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/GroupIdRecordDaoImpl.java
@@ -1,15 +1,20 @@
 package org.orcid.persistence.dao.impl;
 
+import java.util.Date;
 import java.util.List;
 
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
 
 import org.orcid.persistence.dao.GroupIdRecordDao;
 import org.orcid.persistence.jpa.entities.GroupIdRecordEntity;
+import org.springframework.beans.factory.annotation.Value;
+
+public class GroupIdRecordDaoImpl extends GenericDaoImpl<GroupIdRecordEntity, Long> implements GroupIdRecordDao {
+
+    @Value("${org.orcid.persistence.groupIdRecord.retry.max:5}")
+    private int maxRetries;
 
-public class GroupIdRecordDaoImpl extends GenericDaoImpl<GroupIdRecordEntity, Long> implements GroupIdRecordDao {    
-
     public GroupIdRecordDaoImpl() {
         super(GroupIdRecordEntity.class);
     }
@@ -73,10 +78,11 @@ public boolean duplicateExists(Long putCode, String groupId) {
     }   
 
     @Override
-    public List<GroupIdRecordEntity> getIssnRecordsSortedById(int batchSize, long initialId) {
-        Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g LEFT OUTER JOIN invalid_issn_group_id_record p ON g.id = p.id where p.id IS NULL AND g.group_id like 'issn:%' and g.id > :initialId order by g.id", GroupIdRecordEntity.class);
-        query.setParameter("initialId", initialId);
-        query.setMaxResults(batchSize);        
+    public List<GroupIdRecordEntity> getIssnRecordsSortedBySyncDate(int batchSize, Date syncTime) {
+        Query query = entityManager.createNativeQuery("SELECT * FROM group_id_record g WHERE g.issn_loader_fail_count < :max AND g.group_id LIKE 'issn:%' AND (g.sync_date is null OR g.sync_date < :syncTime) ORDER BY g.sync_date", GroupIdRecordEntity.class);
+        query.setParameter("max", maxRetries);
+        query.setParameter("syncTime", syncTime);
+        query.setMaxResults(batchSize);
         return query.getResultList();
     }
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/GroupIdRecordEntity.java
@@ -7,6 +7,7 @@
 import javax.persistence.Id;
 import javax.persistence.SequenceGenerator;
 import javax.persistence.Table;
+import java.util.Date;
 
 /**
  * The persistent class for the group_id_record database table.
@@ -26,7 +27,13 @@ public class GroupIdRecordEntity extends SourceAwareEntity<Long> implements Comp
 
     private String groupDescription;
 
-    private String groupType;    
+    private String groupType;
+
+    private Integer issnLoaderFailCount;
+
+    private String failReason;
+
+    private Date syncDate;
 
     @Id
     @GeneratedValue(strategy = GenerationType.AUTO, generator = "group_id_record_seq")
@@ -75,6 +82,33 @@ public void setGroupType(String groupType) {
         this.groupType = groupType;
     }
 
+    @Column(name = "issn_loader_fail_count")
+    public Integer getIssnLoaderFailCount() {
+        return issnLoaderFailCount;
+    }
+
+    public void setIssnLoaderFailCount(Integer issnLoaderFailCount) {
+        this.issnLoaderFailCount = issnLoaderFailCount;
+    }
+
+    @Column(name = "fail_reason")
+    public String getFailReason() {
+        return failReason;
+    }
+
+    public void setFailReason(String failReason) {
+        this.failReason = failReason;
+    }
+
+    @Column(name = "sync_date")
+    public Date getSyncDate() {
+        return syncDate;
+    }
+
+    public void setSyncDate(Date syncDate) {
+        this.syncDate = syncDate;
+    }
+
     @Override
     public int compareTo(GroupIdRecordEntity o) {
         return 0;

diff --git a/...ence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java b/...ence/src/main/java/org/orcid/persistence/jpa/entities/InvalidIssnGroupIdRecordEntity.java
diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml
@@ -89,8 +89,7 @@
 		<class>org.orcid.persistence.jpa.entities.RejectedGroupingSuggestionEntity</class>
 		<class>org.orcid.persistence.jpa.entities.ValidatedPublicProfileEntity</class>
 		<class>org.orcid.persistence.jpa.entities.MemberOBOWhitelistedClientEntity</class>
-		<class>org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity</class>
-		<class>org.orcid.persistence.jpa.entities.OrgImportLogEntity</class>		
+		<class>org.orcid.persistence.jpa.entities.OrgImportLogEntity</class>
 
 		<!-- Statistics -->
 		<class>org.orcid.statistics.jpa.entities.StatisticValuesEntity</class>

diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml
@@ -391,4 +391,5 @@
   <include file="/db/updates/org_disambiguated_extra_columns.xml" />
   <include file="/db/updates/varchar_to_text_hsqldb.xml" />
   <include file="/db/updates/identifier-types/identifier-type-52-RAiD.xml" />
+  <include file="/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml" />
 </databaseChangeLog>
diff --git a/...main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml b/...main/resources/db/updates/add_fail_count_fail_reason_and_sync_date_to_group_id_record.xml
@@ -0,0 +1,50 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+    <changeSet id="ADD-ISSN_LOADER_FAIL_COUNT" author="Angel Montenegro">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <columnExists tableName="group_id_record" columnName="issn_loader_fail_count"/>
+            </not>
+        </preConditions>
+        <sql>ALTER TABLE group_id_record ADD COLUMN issn_loader_fail_count INTEGER default 0;</sql>
+    </changeSet>
+
+    <changeSet id="ADD-FAIL_REASON" author="Angel Montenegro">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <columnExists tableName="group_id_record" columnName="fail_reason"/>
+            </not>
+        </preConditions>
+        <sql>ALTER TABLE group_id_record ADD COLUMN fail_reason VARCHAR(50);</sql>
+    </changeSet>
+
+    <changeSet id="ADD-SYNC_DATE" author="Angel Montenegro">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <columnExists tableName="group_id_record" columnName="sync_date"/>
+            </not>
+        </preConditions>
+        <sql>ALTER TABLE group_id_record ADD COLUMN sync_date timestamp;</sql>
+    </changeSet>
+
+    <changeSet id="ISSN_LOADER_FAIL_COUNT-INDEX" author="Angel Montenegro">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <indexExists indexName="group_id_record_issn_loader_fail_count_index" tableName="group_id_record" />
+            </not>
+        </preConditions>
+        <sql>create index group_id_record_issn_loader_fail_count_index on group_id_record(issn_loader_fail_count);</sql>
+    </changeSet>
+
+    <changeSet id="ISSN_LOADER_SYNC_DATE-INDEX" author="Angel Montenegro">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <indexExists indexName="group_id_record_sync_date_index" tableName="group_id_record" />
+            </not>
+        </preConditions>
+        <sql>create index group_id_record_sync_date_index on group_id_record(sync_date);</sql>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml
@@ -271,10 +271,6 @@
         <property name="entityManager" ref="entityManagerReadOnly" />
     </bean>
 
-    <bean id="invalidIssnGroupIdRecordDao" class="org.orcid.persistence.dao.impl.GenericDaoImpl">
-        <constructor-arg value="org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity" />
-    </bean>
-
     <bean id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean" depends-on="liquibase">
         <property name="jpaVendorAdapter" ref="jpaVendorAdapter" />
         <property name="dataSource" ref="\${org.orcid.persistence.db.dataSource}" />

diff --git a/...eduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java b/...eduler-web/src/main/java/org/orcid/scheduler/loader/manager/impl/IssnLoadManagerImpl.java
@@ -36,7 +36,6 @@ public void loadIssn() {
             LOGGER.info("Load ISSN  for client : " + issnSource);
             issnLoadSource.loadIssn(issnSource);
             slackManager.sendAlert("Issn  succesfully updated for client " + issnSource, slackChannel, slackUser);
-
         } catch (Exception ex) {
             LOGGER.error("Error when running ISSN for client" + issnSource, ex);
             slackManager.sendAlert("Error when running ISSN for client " + issnSource, slackChannel, slackUser);

diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java
@@ -2,6 +2,7 @@
 
 import java.io.IOException;
 import java.net.URISyntaxException;
+import java.util.Date;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -15,11 +16,9 @@
 import org.orcid.core.groupIds.issn.IssnData;
 import org.orcid.core.groupIds.issn.IssnValidator;
 import org.orcid.persistence.dao.ClientDetailsDao;
-import org.orcid.persistence.dao.GenericDao;
 import org.orcid.persistence.dao.GroupIdRecordDao;
 import org.orcid.persistence.jpa.entities.ClientDetailsEntity;
 import org.orcid.persistence.jpa.entities.GroupIdRecordEntity;
-import org.orcid.persistence.jpa.entities.InvalidIssnGroupIdRecordEntity;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -35,17 +34,14 @@ public class IssnLoadSource {
     @Value("${org.orcid.scheduler.issnLoadSource.batchSize:5000}")
     private int batchSize;
 
-    @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:30000}")
+    @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:10000}")
     private int waitBetweenBatches;
 
     @Resource
     private GroupIdRecordDao groupIdRecordDao;
 
     @Resource(name="groupIdRecordDaoReadOnly")
     private GroupIdRecordDao groupIdRecordDaoReadOnly;
-
-    @Resource
-    private GenericDao<InvalidIssnGroupIdRecordEntity, Long> invalidIssnGroupIdRecordDao;
 
     @Resource(name="clientDetailsDaoReadOnly")
     private ClientDetailsDao clientDetailsDaoReadOnly;
@@ -72,10 +68,10 @@ public void loadIssn(String issnSource) {
     }
 
     private void updateIssnGroupIdRecords() {
-        Long nextBatchStartId = 0L;
+        Date startTime = new Date();
         // Get the first batch of issn's
-        LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId + " batch size: " + batchSize);
-        List<GroupIdRecordEntity> issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId);
+        LOG.info("Running the process to load ISSN info, starting time: " + startTime + " batch size: " + batchSize);
+        List<GroupIdRecordEntity> issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime);
         int batchCount = 0;
         int total = 0;
         while (!issnEntities.isEmpty()) {
@@ -95,23 +91,23 @@ private void updateIssnGroupIdRecords() {
                     } catch(TooManyRequestsException tmre) {
                         //TODO: We are being rate limited, we have to pause
                         LOG.warn("We are being rate limited by the issn portal");
-                        recordFailure(issnEntity.getId(), "RATE_LIMIT reached");
+                        recordFailure(issnEntity, "RATE_LIMIT reached");
                     } catch(UnexpectedResponseCodeException urce) {
                         LOG.warn("Unexpected response code {} for issn {}", urce.getReceivedCode(), issn);
-                        recordFailure(issnEntity.getId(), "Unexpected response code " + urce.getReceivedCode());
+                        recordFailure(issnEntity, "Unexpected response code " + urce.getReceivedCode());
                     } catch (IOException e) {
                         LOG.warn("IOException for issn {}", issn);
-                        recordFailure(issnEntity.getId(), "IOException");
+                        recordFailure(issnEntity, "IOException");
                     } catch (URISyntaxException e) {
                         LOG.warn("URISyntaxException for issn {}", issn);
-                        recordFailure(issnEntity.getId(), "URISyntaxException");
+                        recordFailure(issnEntity, "URISyntaxException");
                     } catch (InterruptedException e) {
                         LOG.warn("InterruptedException for issn {}", issn);
-                        recordFailure(issnEntity.getId(), "InterruptedException");
+                        recordFailure(issnEntity, "InterruptedException");
                     }
                 } else {
                     LOG.info("Issn for group record {} not valid: {}", issnEntity.getId(), issnEntity.getGroupId());
-                    recordFailure(issnEntity.getId(), "Invalid record");
+                    recordFailure(issnEntity, "Invalid record");
                 }                
                 try {
                     // Lets sleep for 30 secs after processing one batch
@@ -125,21 +121,22 @@ private void updateIssnGroupIdRecords() {
                     // TODO Auto-generated catch block
                     LOG.warn("Exception while pausing the issn loader", e);                    
                 }
-
-                if (issnEntity.getId() > nextBatchStartId) {
-                    nextBatchStartId = issnEntity.getId();
-                }
             }
-            LOG.info("Loading batch of ISSN's, starting id: " + nextBatchStartId);
-            issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedById(batchSize, nextBatchStartId);
+            LOG.info("Loading next batch of ISSN's");
+            issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime);
         }
+        LOG.info("All ISSN records processed");
     }
 
-    private void recordFailure(Long id, String notes) {
-        InvalidIssnGroupIdRecordEntity invalidIssn = new InvalidIssnGroupIdRecordEntity();
-        invalidIssn.setId(id);
-        invalidIssn.setNotes(notes);
-        invalidIssnGroupIdRecordDao.persist(invalidIssn);
+    private void recordFailure(GroupIdRecordEntity issnEntity, String notes) {
+        issnEntity.setFailReason(notes);
+        issnEntity.setSyncDate(new Date());
+        if(issnEntity.getIssnLoaderFailCount() == null) {
+            issnEntity.setIssnLoaderFailCount(1);
+        } else {
+            issnEntity.setIssnLoaderFailCount(issnEntity.getIssnLoaderFailCount() + 1);
+        }
+        groupIdRecordDao.merge(issnEntity);
     }
 
     private void updateIssnEntity(GroupIdRecordEntity issnEntity, IssnData issnData) {
@@ -149,10 +146,13 @@ private void updateIssnEntity(GroupIdRecordEntity issnEntity, IssnData issnData)
         if(!StringUtils.equals(currentGroupName, updatedGroupName)) {
             issnEntity.setGroupName(updatedGroupName);            
             issnEntity.setClientSourceId(orcidSource.getId());
+            issnEntity.setSyncDate(new Date());
             LOG.info("Updating Group id: " + issnEntity.getGroupId() +  " | current group name: " + currentGroupName +  " | group name  to be updated: " + issnEntity.getGroupName());
             groupIdRecordDao.merge(issnEntity);
         } else {
-            LOG.info("Group id: " + issnEntity.getGroupId() + " is up to date");
+            issnEntity.setSyncDate(new Date());
+            groupIdRecordDao.merge(issnEntity);
+            LOG.debug("Group id: " + issnEntity.getGroupId() + " is up to date");
         }
     }