Issn loader improvements

orcid-core/src/main/java/org/orcid/core/exception/BannedException.java

@@ -0,0 +1,17 @@
+package org.orcid.core.exception;
+
+import java.util.Map;
+
+public class BannedException extends ApplicationException {
+
+    private static final long serialVersionUID = 1L;
+
+    public BannedException() {
+        super();
+    }
+
+    public BannedException(Map<String, String> params) {
+        super(params);
+    }
+
+}

orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java

@@ -12,6 +12,7 @@
 import org.codehaus.jettison.json.JSONException;
 import org.codehaus.jettison.json.JSONObject;
 import org.eclipse.jetty.http.HttpStatus;
+import org.orcid.core.exception.BannedException;
 import org.orcid.core.exception.TooManyRequestsException;
 import org.orcid.core.exception.UnexpectedResponseCodeException;
 import org.orcid.core.utils.http.HttpRequestUtils;
@@ -36,19 +37,24 @@ public class IssnClient {
     @Resource
     private HttpRequestUtils httpRequestUtils;
 
-    public IssnData getIssnData(String issn) throws TooManyRequestsException, UnexpectedResponseCodeException, IOException, URISyntaxException, InterruptedException {
+    public IssnData getIssnData(String issn) throws BannedException, TooManyRequestsException, UnexpectedResponseCodeException, IOException, URISyntaxException, InterruptedException, JSONException {
         if(StringUtils.isEmpty(issn)) {
             return null;
         }
         LOG.debug("Extracting ISSN for " +  issn);
         String json = getJsonDataFromIssnPortal(issn.toUpperCase());
         try {
-            if (json != null) {
-                IssnData data = extractIssnData(issn.toUpperCase(), json);
-                data.setIssn(issn);
-                return data;
-            } else {
+            IssnData data = extractIssnData(issn.toUpperCase(), json);
+            data.setIssn(issn);
+            return data;
+        } catch (JSONException e) {
+            LOG.warn("Error extracting issn data from json returned from issn portal "+ issn);
+            if(json == null) {
                 return null;
+            } else if(json.contains("you have been banned")) {
+                throw new BannedException();
+            } else {
+                throw e;
             }
         } catch (Exception e) {
             LOG.warn("Error extracting issn data from json returned from issn portal "+ issn);

orcid-core/src/main/java/org/orcid/core/manager/impl/GroupIdRecordManagerImpl.java

@@ -6,6 +6,7 @@
 
 import javax.annotation.Resource;
 
+import org.codehaus.jettison.json.JSONException;
 import org.orcid.core.exception.*;
 import org.orcid.core.groupIds.issn.IssnClient;
 import org.orcid.core.groupIds.issn.IssnData;
@@ -149,6 +150,9 @@ private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) {
         } catch (InterruptedException e) {
             LOG.warn("InterruptedException for issn {}", issn);
             throw new InvalidIssnException();
+        } catch(JSONException e) {
+            LOG.warn("JSONException for issn {}", issn, e);
+            throw new InvalidIssnException();
         }
     }
 

orcid-core/src/main/java/org/orcid/core/manager/v3/impl/GroupIdRecordManagerImpl.java

@@ -6,6 +6,7 @@
 
 import javax.annotation.Resource;
 
+import org.codehaus.jettison.json.JSONException;
 import org.orcid.core.exception.*;
 import org.orcid.core.groupIds.issn.IssnClient;
 import org.orcid.core.groupIds.issn.IssnData;
@@ -66,7 +67,7 @@ public GroupIdRecord createGroupIdRecord(GroupIdRecord groupIdRecord) {
     }
 
     @Override
-    public GroupIdRecord createOrcidSourceIssnGroupIdRecord(String groupId, String issn) {
+    public GroupIdRecord createOrcidSourceIssnGroupIdRecord(String groupId, String issn) throws TooManyRequestsException, BannedException {
         GroupIdRecord issnRecord = createIssnGroupIdRecord(groupId, issn);
         GroupIdRecordEntity entity = jpaJaxbGroupIdRecordAdapter.toGroupIdRecordEntity(issnRecord);
         entity.setClientSourceId(orcidSourceClientDetailsId);
@@ -114,7 +115,7 @@ public void deleteGroupIdRecord(Long putCode) {
         }
     }
 
-    private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) {
+    private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) throws TooManyRequestsException, BannedException {
         if (!issnValidator.issnValid(issn)) {
             throw new InvalidIssnException();
         }
@@ -148,6 +149,9 @@ private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) {
         } catch (InterruptedException e) {
             LOG.warn("InterruptedException for issn {}", issn);
             throw new InvalidIssnException();
+        } catch(JSONException e) {
+            LOG.warn("JSONException for issn {}", issn);
+            throw new InvalidIssnException();
         }
     }
 

orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java

@@ -1,7 +1,7 @@
 package org.orcid.scheduler.loader.source.issn;
 
 import java.io.IOException;
-import java.net.URISyntaxException;
+import java.net.*;
 import java.util.Date;
 import java.util.List;
 import java.util.regex.Matcher;
@@ -10,6 +10,8 @@
 import javax.annotation.Resource;
 
 import org.apache.commons.lang3.StringUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.orcid.core.exception.BannedException;
 import org.orcid.core.exception.TooManyRequestsException;
 import org.orcid.core.exception.UnexpectedResponseCodeException;
 import org.orcid.core.groupIds.issn.IssnClient;
@@ -19,6 +21,7 @@
 import org.orcid.persistence.dao.GroupIdRecordDao;
 import org.orcid.persistence.jpa.entities.ClientDetailsEntity;
 import org.orcid.persistence.jpa.entities.GroupIdRecordEntity;
+import org.orcid.utils.alerting.SlackManager;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
@@ -36,7 +39,10 @@ public class IssnLoadSource {
 
     @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:10000}")
     private int waitBetweenBatches;
-
+
+    @Value("${org.orcid.scheduler.issnLoadSource.rateLimit.pause:600000}")
+    private int pause;
+
     @Resource
     private GroupIdRecordDao groupIdRecordDao;
 
@@ -53,6 +59,9 @@ public class IssnLoadSource {
 
     @Resource
     private IssnClient issnClient;
+
+    @Resource
+    private SlackManager slackManager;
 
     public void loadIssn(String issnSource) {
 
@@ -74,6 +83,7 @@ private void updateIssnGroupIdRecords() {
         List<GroupIdRecordEntity> issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime);
         int batchCount = 0;
         int total = 0;
+
         while (!issnEntities.isEmpty()) {
             for (GroupIdRecordEntity issnEntity : issnEntities) {
                 LOG.info("Processing entity {}", new Object[]{ issnEntity.getId() });
@@ -89,9 +99,29 @@ private void updateIssnGroupIdRecords() {
                                     new Object[]{issnEntity.getId(), issnEntity.getGroupId(), Integer.toString(total)});
                         }
                     } catch(TooManyRequestsException tmre) {
-                        //TODO: We are being rate limited, we have to pause
+                        //We are being rate limited, we have to pause for 'pause' minutes
                         LOG.warn("We are being rate limited by the issn portal");
                         recordFailure(issnEntity, "RATE_LIMIT reached");
+                        if(pause() != 1) {
+                            LOG.warn("Unable to pause, finishing the process");
+                            return;
+                        }
+                    } catch(BannedException be) {
+                        LOG.error("We have been banned from the issn portal, the sync process will finish now");
+                        try {
+                            InetAddress id = InetAddress.getLocalHost();
+                            slackManager.sendSystemAlert("We have bee banned from the issn portal on " + id.getHostName());
+                        } catch(UnknownHostException uhe) {
+                            // Lets try to get the IP address
+                            try(final DatagramSocket socket = new DatagramSocket()){
+                                socket.connect(InetAddress.getByName("8.8.8.8"), 10002);
+                                String ip = socket.getLocalAddress().getHostAddress();
+                                slackManager.sendSystemAlert("We have bee banned from the issn portal on " + ip);
+                            } catch(SocketException | UnknownHostException se) {
+                                slackManager.sendSystemAlert("We have bee banned from the issn portal on - Couldn't identify the machine");
+                            }
+                        }
+                        return;
                     } catch(UnexpectedResponseCodeException urce) {
                         LOG.warn("Unexpected response code {} for issn {}", urce.getReceivedCode(), issn);
                         recordFailure(issnEntity, "Unexpected response code " + urce.getReceivedCode());
@@ -104,6 +134,9 @@ private void updateIssnGroupIdRecords() {
                     } catch (InterruptedException e) {
                         LOG.warn("InterruptedException for issn {}", issn);
                         recordFailure(issnEntity, "InterruptedException");
+                    } catch(JSONException e) {
+                        LOG.warn("InterruptedException for issn {}", issn);
+                        recordFailure(issnEntity, "InterruptedException");
                     }
                 } else {
                     LOG.info("Issn for group record {} not valid: {}", issnEntity.getId(), issnEntity.getGroupId());
@@ -164,4 +197,15 @@ private String getIssn(GroupIdRecordEntity issnEntity) {
         return null;
     }
 
+    private int pause() {
+        try {
+            LOG.warn("Pause do to rate limit");
+            Thread.sleep(pause);
+            return 1;
+        } catch (InterruptedException e) {
+            LOG.warn("Unable to pause", e);
+            return -1;
+        }
+    }
+
 }

orcid-utils/src/main/java/org/orcid/utils/alerting/impl/SlackManagerImpl.java

@@ -25,7 +25,7 @@
  */
 public class SlackManagerImpl implements SlackManager {
 
-    @Value("${org.orcid.core.slack.webhookUrl:}")
+    @Value("${org.orcid.core.slack.webhookUrl}")
     private String webhookUrl;
 
     @Value("${org.orcid.core.slack.channel}")