diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 4f262838c6..9e8373dfdc 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -5,7 +5,9 @@ import java.time.LocalDate; import java.util.HashMap; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Resource; import javax.servlet.FilterChain; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; @@ -25,6 +27,8 @@ import org.orcid.persistence.jpa.entities.ProfileEntity; import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; import org.orcid.utils.email.MailGunManager; +import org.orcid.utils.panoply.PanoplyPapiDailyRateExceededItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -65,15 +69,21 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Autowired private EmailManager emailManager; + + @Resource + private PanoplyRedshiftClient panoplyClient; - @Value("${rate.limit.anonymous.requests:1}") + @Value("${rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; - @Value("${rate.limit.known.requests:2}") + @Value("${rate.limit.known.requests:40000}") private int knownRequestLimit; @Value("${rate.limit.enabled:false}") private boolean enableRateLimiting; + + @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") + private boolean enablePanoplyPapiExceededRateInProduction; private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + "You can increase your daily quota by registering for and using Public API client credentials " @@ -112,10 +122,15 @@ private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpSe // update the request count only when limit not exceeded ? rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, false); + if (rateLimitEntity.getRequestCount() == knownRequestLimit && enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setIpAddress(ipAddress); + item.setRequestDate(rateLimitEntity.getRequestDate()); + setPapiRateExceededItemInPanoply(item); + } if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { if (rateLimitEntity.getRequestCount() >= anonymousRequestLimit) { httpServletResponse.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - // Use only one writer call if (!httpServletResponse.isCommitted()) { try (PrintWriter writer = httpServletResponse.getWriter()) { writer.write(TOO_MANY_REQUESTS_MSG); @@ -144,7 +159,7 @@ private void rateLimitClientRequest(String clientId, LocalDate today) { if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { // email the client first time the limit is reached if (rateLimitEntity.getRequestCount() == knownRequestLimit) { - sendEmail(clientId); + sendEmail(clientId, rateLimitEntity.getRequestDate()); } } // update the request count @@ -159,6 +174,7 @@ private void rateLimitClientRequest(String clientId, LocalDate today) { rateLimitEntity.setRequestDate(today); papiRateLimitingDao.persist(rateLimitEntity); } + } @@ -174,7 +190,7 @@ private Map createTemplateParams(String clientId, String clientN return templateParams; } - private void sendEmail(String clientId) { + private void sendEmail(String clientId, LocalDate requestDate) { ClientDetailsEntity clientDetailsEntity = clientDetailsEntityCacheManager.retrieve(clientId); ProfileEntity profile = profileDao.find(clientDetailsEntity.getGroupProfileId()); String emailName = recordNameManager.deriveEmailFriendlyName(profile.getId()); @@ -183,15 +199,43 @@ private void sendEmail(String clientId) { String body = templateManager.processTemplate("bad_orgs_email.ftl", templateParams); // Generate html from template String html = templateManager.processTemplate("bad_orgs_email_html.ftl", templateParams); + String email = emailManager.findPrimaryEmail(profile.getId()).getEmail(); LOG.info("text email={}", body); LOG.info("html email={}", html); + if (enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setClientId(clientId); + item.setOrcid(profile.getId()); + item.setEmail(email); + item.setRequestDate(requestDate); + setPapiRateExceededItemInPanoply(item); + } // Send the email - boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, emailManager.findPrimaryEmail(profile.getId()).getEmail(), SUBJECT, body, html); + boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html); if (!mailSent) { throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); } } + + + private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) { + //Store the rate exceeded item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyPapiDailyRateExceeded(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store the rateExceededItem to panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); + } } diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java new file mode 100644 index 0000000000..9761ed95f4 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java @@ -0,0 +1,49 @@ +package org.orcid.utils.panoply; + +import java.time.LocalDate; + +public class PanoplyPapiDailyRateExceededItem { + private String ipAddress; + private String clientId; + private String email; + private String orcid; + private LocalDate requestDate; + + + public String getIpAddress() { + return ipAddress; + } + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + public String getClientId() { + return clientId; + } + public void setClientId(String clientId) { + this.clientId = clientId; + } + public String getEmail() { + return email; + } + public void setEmail(String email) { + this.email = email; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + public LocalDate getRequestDate() { + return requestDate; + } + public void setRequestDate(LocalDate requestDate) { + this.requestDate = requestDate; + } + + @Override + public String toString() { + return "PanoplyPapiDailyRateExceededItem{" + "ipAddress=" + ipAddress + ", clientId='" + clientId + '\'' + ", email='" + email + '\'' + ", orcid='" + orcid + '\'' + + ", requestDate='" + requestDate + '\'' + '}'; + } +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java index 77c5db8ef1..a7b0d24002 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -31,5 +31,11 @@ public int addPanoplyDeletedItem(PanoplyDeletedItem item) { return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); } - + + public int addPanoplyPapiDailyRateExceeded(PanoplyPapiDailyRateExceededItem item) { + LOG.debug("Adding papi daily rate exceeded item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_papi_daily_rate_exceeded (ip_address, orcid, client_id, email, request_date) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getIpAddress(), item.getOrcid(), item.getClientId(), item.getEmail(), item.getRequestDate()); + } + }