diff --git a/orcid-core/src/main/java/org/orcid/core/exception/BannedException.java b/orcid-core/src/main/java/org/orcid/core/exception/BannedException.java new file mode 100644 index 00000000000..e5c7b1f90bc --- /dev/null +++ b/orcid-core/src/main/java/org/orcid/core/exception/BannedException.java @@ -0,0 +1,17 @@ +package org.orcid.core.exception; + +import java.util.Map; + +public class BannedException extends ApplicationException { + + private static final long serialVersionUID = 1L; + + public BannedException() { + super(); + } + + public BannedException(Map params) { + super(params); + } + +} diff --git a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java index 8df8e935147..1a46c7bad55 100644 --- a/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java +++ b/orcid-core/src/main/java/org/orcid/core/groupIds/issn/IssnClient.java @@ -12,6 +12,7 @@ import org.codehaus.jettison.json.JSONException; import org.codehaus.jettison.json.JSONObject; import org.eclipse.jetty.http.HttpStatus; +import org.orcid.core.exception.BannedException; import org.orcid.core.exception.TooManyRequestsException; import org.orcid.core.exception.UnexpectedResponseCodeException; import org.orcid.core.utils.http.HttpRequestUtils; @@ -36,19 +37,24 @@ public class IssnClient { @Resource private HttpRequestUtils httpRequestUtils; - public IssnData getIssnData(String issn) throws TooManyRequestsException, UnexpectedResponseCodeException, IOException, URISyntaxException, InterruptedException { + public IssnData getIssnData(String issn) throws BannedException, TooManyRequestsException, UnexpectedResponseCodeException, IOException, URISyntaxException, InterruptedException, JSONException { if(StringUtils.isEmpty(issn)) { return null; } LOG.debug("Extracting ISSN for " + issn); String json = getJsonDataFromIssnPortal(issn.toUpperCase()); try { - if (json != null) { - IssnData data = extractIssnData(issn.toUpperCase(), json); - data.setIssn(issn); - return data; - } else { + IssnData data = extractIssnData(issn.toUpperCase(), json); + data.setIssn(issn); + return data; + } catch (JSONException e) { + LOG.warn("Error extracting issn data from json returned from issn portal "+ issn); + if(json == null) { return null; + } else if(json.contains("you have been banned")) { + throw new BannedException(); + } else { + throw e; } } catch (Exception e) { LOG.warn("Error extracting issn data from json returned from issn portal "+ issn); diff --git a/orcid-core/src/main/java/org/orcid/core/manager/impl/GroupIdRecordManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/manager/impl/GroupIdRecordManagerImpl.java index 6e667415956..7eb87b7ad89 100644 --- a/orcid-core/src/main/java/org/orcid/core/manager/impl/GroupIdRecordManagerImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/manager/impl/GroupIdRecordManagerImpl.java @@ -6,6 +6,7 @@ import javax.annotation.Resource; +import org.codehaus.jettison.json.JSONException; import org.orcid.core.exception.*; import org.orcid.core.groupIds.issn.IssnClient; import org.orcid.core.groupIds.issn.IssnData; @@ -149,6 +150,9 @@ private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) { } catch (InterruptedException e) { LOG.warn("InterruptedException for issn {}", issn); throw new InvalidIssnException(); + } catch(JSONException e) { + LOG.warn("JSONException for issn {}", issn, e); + throw new InvalidIssnException(); } } diff --git a/orcid-core/src/main/java/org/orcid/core/manager/v3/impl/GroupIdRecordManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/manager/v3/impl/GroupIdRecordManagerImpl.java index e6b251d73cf..82f7435983a 100644 --- a/orcid-core/src/main/java/org/orcid/core/manager/v3/impl/GroupIdRecordManagerImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/manager/v3/impl/GroupIdRecordManagerImpl.java @@ -6,6 +6,7 @@ import javax.annotation.Resource; +import org.codehaus.jettison.json.JSONException; import org.orcid.core.exception.*; import org.orcid.core.groupIds.issn.IssnClient; import org.orcid.core.groupIds.issn.IssnData; @@ -66,7 +67,7 @@ public GroupIdRecord createGroupIdRecord(GroupIdRecord groupIdRecord) { } @Override - public GroupIdRecord createOrcidSourceIssnGroupIdRecord(String groupId, String issn) { + public GroupIdRecord createOrcidSourceIssnGroupIdRecord(String groupId, String issn) throws TooManyRequestsException, BannedException { GroupIdRecord issnRecord = createIssnGroupIdRecord(groupId, issn); GroupIdRecordEntity entity = jpaJaxbGroupIdRecordAdapter.toGroupIdRecordEntity(issnRecord); entity.setClientSourceId(orcidSourceClientDetailsId); @@ -114,7 +115,7 @@ public void deleteGroupIdRecord(Long putCode) { } } - private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) { + private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) throws TooManyRequestsException, BannedException { if (!issnValidator.issnValid(issn)) { throw new InvalidIssnException(); } @@ -148,6 +149,9 @@ private GroupIdRecord createIssnGroupIdRecord(String groupId, String issn) { } catch (InterruptedException e) { LOG.warn("InterruptedException for issn {}", issn); throw new InvalidIssnException(); + } catch(JSONException e) { + LOG.warn("JSONException for issn {}", issn); + throw new InvalidIssnException(); } } diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java index a4e88d73fb9..f59ea373cef 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/issn/IssnLoadSource.java @@ -1,7 +1,7 @@ package org.orcid.scheduler.loader.source.issn; import java.io.IOException; -import java.net.URISyntaxException; +import java.net.*; import java.util.Date; import java.util.List; import java.util.regex.Matcher; @@ -10,6 +10,8 @@ import javax.annotation.Resource; import org.apache.commons.lang3.StringUtils; +import org.codehaus.jettison.json.JSONException; +import org.orcid.core.exception.BannedException; import org.orcid.core.exception.TooManyRequestsException; import org.orcid.core.exception.UnexpectedResponseCodeException; import org.orcid.core.groupIds.issn.IssnClient; @@ -19,6 +21,7 @@ import org.orcid.persistence.dao.GroupIdRecordDao; import org.orcid.persistence.jpa.entities.ClientDetailsEntity; import org.orcid.persistence.jpa.entities.GroupIdRecordEntity; +import org.orcid.utils.alerting.SlackManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -36,7 +39,10 @@ public class IssnLoadSource { @Value("${org.orcid.scheduler.issnLoadSource.waitBetweenBatches:10000}") private int waitBetweenBatches; - + + @Value("${org.orcid.scheduler.issnLoadSource.rateLimit.pause:600000}") + private int pause; + @Resource private GroupIdRecordDao groupIdRecordDao; @@ -53,6 +59,9 @@ public class IssnLoadSource { @Resource private IssnClient issnClient; + + @Resource + private SlackManager slackManager; public void loadIssn(String issnSource) { @@ -74,6 +83,7 @@ private void updateIssnGroupIdRecords() { List issnEntities = groupIdRecordDaoReadOnly.getIssnRecordsSortedBySyncDate(batchSize, startTime); int batchCount = 0; int total = 0; + while (!issnEntities.isEmpty()) { for (GroupIdRecordEntity issnEntity : issnEntities) { LOG.info("Processing entity {}", new Object[]{ issnEntity.getId() }); @@ -89,9 +99,29 @@ private void updateIssnGroupIdRecords() { new Object[]{issnEntity.getId(), issnEntity.getGroupId(), Integer.toString(total)}); } } catch(TooManyRequestsException tmre) { - //TODO: We are being rate limited, we have to pause + //We are being rate limited, we have to pause for 'pause' minutes LOG.warn("We are being rate limited by the issn portal"); recordFailure(issnEntity, "RATE_LIMIT reached"); + if(pause() != 1) { + LOG.warn("Unable to pause, finishing the process"); + return; + } + } catch(BannedException be) { + LOG.error("We have been banned from the issn portal, the sync process will finish now"); + try { + InetAddress id = InetAddress.getLocalHost(); + slackManager.sendSystemAlert("We have bee banned from the issn portal on " + id.getHostName()); + } catch(UnknownHostException uhe) { + // Lets try to get the IP address + try(final DatagramSocket socket = new DatagramSocket()){ + socket.connect(InetAddress.getByName("8.8.8.8"), 10002); + String ip = socket.getLocalAddress().getHostAddress(); + slackManager.sendSystemAlert("We have bee banned from the issn portal on " + ip); + } catch(SocketException | UnknownHostException se) { + slackManager.sendSystemAlert("We have bee banned from the issn portal on - Couldn't identify the machine"); + } + } + return; } catch(UnexpectedResponseCodeException urce) { LOG.warn("Unexpected response code {} for issn {}", urce.getReceivedCode(), issn); recordFailure(issnEntity, "Unexpected response code " + urce.getReceivedCode()); @@ -104,6 +134,9 @@ private void updateIssnGroupIdRecords() { } catch (InterruptedException e) { LOG.warn("InterruptedException for issn {}", issn); recordFailure(issnEntity, "InterruptedException"); + } catch(JSONException e) { + LOG.warn("InterruptedException for issn {}", issn); + recordFailure(issnEntity, "InterruptedException"); } } else { LOG.info("Issn for group record {} not valid: {}", issnEntity.getId(), issnEntity.getGroupId()); @@ -164,4 +197,15 @@ private String getIssn(GroupIdRecordEntity issnEntity) { return null; } + private int pause() { + try { + LOG.warn("Pause do to rate limit"); + Thread.sleep(pause); + return 1; + } catch (InterruptedException e) { + LOG.warn("Unable to pause", e); + return -1; + } + } + } diff --git a/orcid-utils/src/main/java/org/orcid/utils/alerting/impl/SlackManagerImpl.java b/orcid-utils/src/main/java/org/orcid/utils/alerting/impl/SlackManagerImpl.java index 14efca1a2b2..478f67d9136 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/alerting/impl/SlackManagerImpl.java +++ b/orcid-utils/src/main/java/org/orcid/utils/alerting/impl/SlackManagerImpl.java @@ -25,7 +25,7 @@ */ public class SlackManagerImpl implements SlackManager { - @Value("${org.orcid.core.slack.webhookUrl:}") + @Value("${org.orcid.core.slack.webhookUrl}") private String webhookUrl; @Value("${org.orcid.core.slack.channel}")