From c4f2119fc988622ccf20093bca3f9f6de9d3d7a2 Mon Sep 17 00:00:00 2001 From: github actions Date: Mon, 4 Nov 2024 06:07:34 +0000 Subject: [PATCH 01/15] v2.67.7 changelog update --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f2df3eacb..d4eac9becb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v2.67.7 - 2024-11-04 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.6...v2.67.7) + +- [#7120](https://github.com/ORCID/ORCID-Source/pull/7120): 9422-email-domains-interstitial-oauth + ## v2.67.6 - 2024-10-31 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.5...v2.67.6) From 1a65b0449955ae47f4e1deef5e4d9d50634b24df Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:29:33 +0000 Subject: [PATCH 02/15] Fixes for papi limit filter and ML start (#7121) * Fixes for papi limit filter and ML start * Added the exception, fixed formatting --------- Co-authored-by: Angel Montenegro --- .../core/oauth/service/OrcidTokenStore.java | 2 + .../service/OrcidTokenStoreServiceImpl.java | 10 +++++ .../orcid-message-listener-web-context.xml | 14 +++---- .../impl/PublicApiDailyRateLimitDaoImpl.java | 2 +- .../resources/orcid-persistence-context.xml | 14 +++---- .../orcid/api/filters/ApiRateLimitFilter.java | 42 ++++++++++++------- 6 files changed, 53 insertions(+), 31 deletions(-) diff --git a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java index 7169054c74..1397e77f4e 100644 --- a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java +++ b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java @@ -25,4 +25,6 @@ public interface OrcidTokenStore extends TokenStore { OAuth2Authentication readAuthenticationFromCachedToken(Map cachedTokenData); void isClientEnabled(String clientId) throws InvalidTokenException; + + String readClientId(String tokenValue); } diff --git a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java index 7d979566bd..19d5045866 100644 --- a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java @@ -72,6 +72,16 @@ public OrcidOauth2TokenDetail readOrcidOauth2TokenDetail(String token) { return orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token); } + @Override + public String readClientId(String token) { + String clientId = null; + OrcidOauth2TokenDetail orcidTokenDetail = orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token); + if(orcidTokenDetail != null) { + clientId = orcidTokenDetail.getClientDetailsId(); + } + return clientId; + } + /** * Read the authentication stored under the specified token value. * diff --git a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml index ac1054dee1..36a8bbddeb 100644 --- a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml +++ b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml @@ -161,13 +161,13 @@ - - - - - - - + + + + + + + diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java index 589811693b..0955b228b3 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java @@ -23,7 +23,7 @@ public PublicApiDailyRateLimitDaoImpl() { @Override public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) { - Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate", + Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p where p.client_id=:clientId and p.request_date=:requestDate", PublicApiDailyRateLimitEntity.class); nativeQuery.setParameter("clientId", clientId); nativeQuery.setParameter("requestDate", requestDate); diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 03fce2ff5d..a96f17593f 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -460,13 +460,13 @@ - - - - - - - + + + + + + + diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 4a6105494d..54a911f8dd 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -20,6 +20,7 @@ import org.orcid.core.manager.impl.OrcidUrlManager; import org.orcid.core.manager.v3.EmailManager; import org.orcid.core.manager.v3.RecordNameManager; +import org.orcid.core.oauth.service.OrcidTokenStore; import org.orcid.core.utils.OrcidRequestUtil; import org.orcid.persistence.dao.ProfileDao; import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; @@ -69,9 +70,12 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Autowired private EmailManager emailManager; - + @Resource - private PanoplyRedshiftClient panoplyClient; + private PanoplyRedshiftClient panoplyClient; + + @Autowired + private OrcidTokenStore orcidTokenStore; @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; @@ -81,9 +85,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Value("${org.orcid.papi.rate.limit.enabled:false}") private boolean enableRateLimiting; - + @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") - private boolean enablePanoplyPapiExceededRateInProduction; + private boolean enablePanoplyPapiExceededRateInProduction; private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + "You can increase your daily quota by registering for and using Public API client credentials " @@ -97,8 +101,16 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl throws ServletException, IOException { LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); if (enableRateLimiting) { - String clientId = orcidSecurityManager.getClientIdFromAPIRequest(); - String ipAddress = OrcidRequestUtil.getIpAddress(httpServletRequest); + String tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim(); + + String ipAddress = httpServletRequest.getRemoteAddr(); + + String clientId = null; + try { + clientId = orcidTokenStore.readClientId(tokenValue); + } catch (Exception ex) { + LOG.error("Exception when trying to get the client id from token value, ignoring and treating as anonymous client", ex); + } boolean isAnonymous = (clientId == null); LocalDate today = LocalDate.now(); @@ -163,17 +175,16 @@ private void rateLimitClientRequest(String clientId, LocalDate today) { } // update the request count rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); - papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity,true); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, true); } else { // create rateLimitEntity = new PublicApiDailyRateLimitEntity(); rateLimitEntity.setClientId(clientId); - rateLimitEntity.setRequestCount(0L); + rateLimitEntity.setRequestCount(1L); rateLimitEntity.setRequestDate(today); papiRateLimitingDao.persist(rateLimitEntity); } - } @@ -212,15 +223,14 @@ private void sendEmail(String clientId, LocalDate requestDate) { } // Send the email - boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html); + boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email, SUBJECT, body, html); if (!mailSent) { throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); } } - - + private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) { - //Store the rate exceeded item in panoply Db without blocking + // Store the rate exceeded item in panoply Db without blocking CompletableFuture.supplyAsync(() -> { try { panoplyClient.addPanoplyPapiDailyRateExceeded(item); @@ -229,9 +239,9 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i LOG.error("Cannot store the rateExceededItem to panoply ", e); return false; } - }).thenAccept(result -> { - if(! result) { - LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + }).thenAccept(result -> { + if (!result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result); } }); From 85cfb05fb8752bcdcc8a32f68c36db9fd8593b10 Mon Sep 17 00:00:00 2001 From: github actions Date: Mon, 4 Nov 2024 17:43:40 +0000 Subject: [PATCH 03/15] v2.67.8 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d4eac9becb..c2f71063e0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.8 - 2024-11-04 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.7...v2.67.8) + ## v2.67.7 - 2024-11-04 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.6...v2.67.7) From 856af1f081160209deb19a0f574ab81ef2e5975a Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Tue, 5 Nov 2024 11:48:50 +0000 Subject: [PATCH 04/15] Papi limit filter NPE when no Authorization header (#7122) * Fixes for papi limit filter and ML start * Added the exception, fixed formatting * Fixed the NPE when no authorization header in the request --- .../orcid/api/filters/ApiRateLimitFilter.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 54a911f8dd..2db61a53b1 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -101,15 +101,19 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl throws ServletException, IOException { LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); if (enableRateLimiting) { - String tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim(); - + String tokenValue = null; + if (httpServletRequest.getHeader("Authorization") != null) { + tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim(); + } String ipAddress = httpServletRequest.getRemoteAddr(); String clientId = null; - try { - clientId = orcidTokenStore.readClientId(tokenValue); - } catch (Exception ex) { - LOG.error("Exception when trying to get the client id from token value, ignoring and treating as anonymous client", ex); + if (tokenValue != null) { + try { + clientId = orcidTokenStore.readClientId(tokenValue); + } catch (Exception ex) { + LOG.error("Exception when trying to get the client id from token value, ignoring and treating as anonymous client", ex); + } } boolean isAnonymous = (clientId == null); LocalDate today = LocalDate.now(); From 690a53a08d2fc4100b4bb9bb1bcdc32103acb357 Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 5 Nov 2024 12:02:23 +0000 Subject: [PATCH 05/15] v2.67.9 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c2f71063e0..171398cdad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.9 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.8...v2.67.9) + ## v2.67.8 - 2024-11-04 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.7...v2.67.8) From c0b8948a54836eab0e3aaab023144c6c9e7be1d3 Mon Sep 17 00:00:00 2001 From: andrej romanov <50377758+auumgn@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:17:41 +0200 Subject: [PATCH 06/15] add null check when fetching emails (#7123) Co-authored-by: Angel Montenegro --- .../web/controllers/ManageProfileController.java | 14 ++++++++------ .../web/controllers/PublicRecordController.java | 11 +++++++---- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java index 72997bfb7b..9c12d1c02f 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java @@ -535,12 +535,14 @@ public ModelAndView confirmDeactivateOrcidAccount(HttpServletRequest request, Ht } org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(v2Emails, emailDomains); // Old emails are missing the source name and id -- assign the user as the source - for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { - if (email.getSource() == null && email.getSourceName() == null) { - String orcid = getCurrentUserOrcid(); - String displayName = getPersonDetails(orcid, true).getDisplayName(); - email.setSource(orcid); - email.setSourceName(displayName); + if (emails.getEmails() != null) { + for (org.orcid.pojo.ajaxForm.Email email : emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + String orcid = getCurrentUserOrcid(); + String displayName = getPersonDetails(orcid, true).getDisplayName(); + email.setSource(orcid); + email.setSourceName(displayName); + } } } return emails; diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java index b078751344..9de6f1207f 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java @@ -205,12 +205,15 @@ PublicRecord getRecord(String orcid) { org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(filteredEmails, emailDomains); // Old emails are missing the source name and id -- assign the user as the source - for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { - if (email.getSource() == null && email.getSourceName() == null) { - email.setSource(orcid); - email.setSourceName(publicRecord.getDisplayName()); + if (emails.getEmails() != null) { + for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + email.setSource(orcid); + email.setSourceName(publicRecord.getDisplayName()); + } } } + publicRecord.setEmails(emails); // Fill external identifiers From 4f76902c36537ac9035bdcfaae0c8f33045d3de3 Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 5 Nov 2024 13:31:30 +0000 Subject: [PATCH 07/15] v2.67.10 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 171398cdad..c12568da3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.10 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.9...v2.67.10) + ## v2.67.9 - 2024-11-05 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.8...v2.67.9) From d6f5e61ec8380aba427b36176e8a38479abd79ed Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Tue, 5 Nov 2024 19:22:21 +0000 Subject: [PATCH 08/15] White list implementation for papi limit (#7126) * Added the white list implementation * kept just 127.0.0.1 as the ip address in the whitelist as default * added the Ip address to the info log for anonymous requests --- .../orcid/api/filters/ApiRateLimitFilter.java | 44 +++++++++++++++++-- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 2db61a53b1..daa982cae3 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -3,7 +3,9 @@ import java.io.IOException; import java.io.PrintWriter; import java.time.LocalDate; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -14,6 +16,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.LocaleUtils; +import org.apache.commons.lang3.StringUtils; import org.orcid.core.manager.ClientDetailsEntityCacheManager; import org.orcid.core.manager.OrcidSecurityManager; import org.orcid.core.manager.TemplateManager; @@ -88,6 +91,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") private boolean enablePanoplyPapiExceededRateInProduction; + + @Value("${org.orcid.papi.rate.limit.ip.whiteSpaceSeparatedWhiteList:127.0.0.1}") + private String papiWhiteSpaceSeparatedWhiteList; private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + "You can increase your daily quota by registering for and using Public API client credentials " @@ -105,7 +111,7 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl if (httpServletRequest.getHeader("Authorization") != null) { tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim(); } - String ipAddress = httpServletRequest.getRemoteAddr(); + String ipAddress = getClientIpAddress(httpServletRequest); String clientId = null; if (tokenValue != null) { @@ -118,9 +124,11 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl boolean isAnonymous = (clientId == null); LocalDate today = LocalDate.now(); - if (isAnonymous) { - LOG.info("ApiRateLimitFilter anonymous request"); - this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + if (isAnonymous ) { + if(!isWhiteListed(ipAddress)) { + LOG.info("ApiRateLimitFilter anonymous request for ip: " + ipAddress); + this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + } } else { LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); @@ -250,5 +258,33 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i }); } + + //gets actual client IP address, using the headers that the proxy server ads + private String getClientIpAddress(HttpServletRequest request) { + String ipAddress = request.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { + ipAddress = request.getHeader("X-REAL-IP"); + } + if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { + ipAddress = request.getRemoteAddr(); + } + if (ipAddress != null && ipAddress.contains(",")) { + ipAddress = ipAddress.split(",")[0].trim(); + } + return ipAddress; + } + + private boolean isWhiteListed(String ipAddress) { + List papiIpWhiteList = null; + if(StringUtils.isNotBlank(papiWhiteSpaceSeparatedWhiteList)) { + papiIpWhiteList = Arrays.asList(papiWhiteSpaceSeparatedWhiteList.split("\\s")); + } + + if(papiIpWhiteList != null) { + return papiIpWhiteList.contains(ipAddress); + + } + return false; + } } From 46e49855912eb349175bd1ae0c07b24ef8fa6fd7 Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 5 Nov 2024 19:38:33 +0000 Subject: [PATCH 09/15] v2.67.11 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c12568da3e..3cd68e158f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.11 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.10...v2.67.11) + ## v2.67.10 - 2024-11-05 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.9...v2.67.10) From 24bee0722df7df8509271d1d7bc1ad7d0df524f3 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:30:00 +0000 Subject: [PATCH 10/15] Fixed typo and the properties to match the papi ones (#7127) --- .../orcid/scheduler/report/PapiDailyLimitReport.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java index 0e63e4f905..898a1896c8 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -37,13 +37,13 @@ public class PapiDailyLimitReport { @Value("${org.orcid.core.orgs.load.slackUser}") private String slackUser; - @Value("${rate.limit.anonymous.requests}") + @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; - @Value("${rate.limit.known.requests}") + @Value("${org.orcid.papi.rate.limit.known.requests:40000}") private int knownRequestLimit; - @Value("${rate.limit.enabled:false}") + @Value("${org.orcid.papi.rate.limit.enabled:false}") private boolean enableRateLimiting; @Autowired @@ -72,12 +72,12 @@ public void papiDailyLimitReport() { LocalDate yesterday = LocalDate.now().minusDays(1); String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\n Current Anonymous Requests Limit: " + anonymousRequestLimit - + "\n Current Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; + + "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; LOG .info(SLACK_INTRO_MSG); slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl); String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit) - + "\n Count of Public API clients that have exceeded the limit: " + + "\nCount of Public API clients that have exceeded the limit: " + papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit); LOG .info(SLACK_STATS_MSG); slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl); From 98baf91c883c9ae0ca4ef7c813daa35318534fdf Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 6 Nov 2024 14:43:57 +0000 Subject: [PATCH 11/15] v2.67.12 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3cd68e158f..aec7e03e17 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.12 - 2024-11-06 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.11...v2.67.12) + ## v2.67.11 - 2024-11-05 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.10...v2.67.11) From 501dab79bb3e6952c67732fa485e623dc7811560 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:07:29 +0000 Subject: [PATCH 12/15] Fixed the 500 when client reached the limit. (#7128) * Fixed the 500 when client reached the limit. * Remove extrawhite space --- .../orcid/core/template/papi_rate_limit_email.ftl | 2 +- .../core/template/papi_rate_limit_email_html.ftl | 3 --- .../org/orcid/api/filters/ApiRateLimitFilter.java | 12 +++++++++--- .../orcid/scheduler/report/PapiDailyLimitReport.java | 4 ++-- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl index 0c4345237c..942550a889 100644 --- a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl @@ -15,5 +15,5 @@ To minimize any disruption to your ORCID integration in the future, we would rec Warm Regards, ORCID Support Team https://support.orcid.org -<@emailMacros.msg "email.common.you_have_received_this_email" /> + <#include "email_footer.ftl"/> diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl index 569cb86ebe..f7d4ebca59 100644 --- a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl @@ -25,9 +25,6 @@

- <@emailMacros.msg "email.common.you_have_received_this_email" /> -

-

<#include "email_footer_html.ftl"/>

diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index daa982cae3..e2f9aae1dd 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -37,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; @@ -79,6 +80,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Autowired private OrcidTokenStore orcidTokenStore; + + @Autowired + private MessageSource messageSource; @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; @@ -202,8 +206,10 @@ private void rateLimitClientRequest(String clientId, LocalDate today) { private Map createTemplateParams(String clientId, String clientName, String emailName, String orcidId) { Map templateParams = new HashMap(); + templateParams.put("messages", messageSource); + templateParams.put("messageArgs", new Object[0]); templateParams.put("clientId", clientId); - templateParams.put("clientId", clientName); + templateParams.put("clientName", clientName); templateParams.put("emailName", emailName); templateParams.put("locale", LocaleUtils.toLocale("en")); templateParams.put("baseUri", orcidUrlManager.getBaseUrl()); @@ -218,9 +224,9 @@ private void sendEmail(String clientId, LocalDate requestDate) { String emailName = recordNameManager.deriveEmailFriendlyName(profile.getId()); Map templateParams = this.createTemplateParams(clientId, clientDetailsEntity.getClientName(), emailName, profile.getId()); // Generate body from template - String body = templateManager.processTemplate("bad_orgs_email.ftl", templateParams); + String body = templateManager.processTemplate("papi_rate_limit_email.ftl", templateParams); // Generate html from template - String html = templateManager.processTemplate("bad_orgs_email_html.ftl", templateParams); + String html = templateManager.processTemplate("papi_rate_limit_email_html.ftl", templateParams); String email = emailManager.findPrimaryEmail(profile.getId()).getEmail(); LOG.info("text email={}", body); diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java index 898a1896c8..73b1b4258c 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -71,8 +71,8 @@ public void papiDailyLimitReport() { if (enableRateLimiting) { LocalDate yesterday = LocalDate.now().minusDays(1); String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; - String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\n Current Anonymous Requests Limit: " + anonymousRequestLimit - + "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; + String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\nCurrent Anonymous Requests Limit: " + anonymousRequestLimit + + "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\nMode: " + mode; LOG .info(SLACK_INTRO_MSG); slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl); From 471cb318737ca495992c538fb6ae8cf20384b858 Mon Sep 17 00:00:00 2001 From: github actions Date: Thu, 7 Nov 2024 17:21:45 +0000 Subject: [PATCH 13/15] v2.67.13 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec7e03e17..5231d45e93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.13 - 2024-11-07 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.12...v2.67.13) + ## v2.67.12 - 2024-11-06 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.11...v2.67.12) From e5a5bfd6076eacd56d606e1e0dd949d8f27635c9 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Thu, 7 Nov 2024 19:53:27 +0000 Subject: [PATCH 14/15] Exception catching for papi limit filter (#7132) * Exception catching for papi limit filter * Logging the email as well --- .../orcid/api/filters/ApiRateLimitFilter.java | 51 ++++++++++--------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index e2f9aae1dd..58779274a4 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -80,7 +80,7 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Autowired private OrcidTokenStore orcidTokenStore; - + @Autowired private MessageSource messageSource; @@ -95,9 +95,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") private boolean enablePanoplyPapiExceededRateInProduction; - + @Value("${org.orcid.papi.rate.limit.ip.whiteSpaceSeparatedWhiteList:127.0.0.1}") - private String papiWhiteSpaceSeparatedWhiteList; + private String papiWhiteSpaceSeparatedWhiteList; private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + "You can increase your daily quota by registering for and using Public API client credentials " @@ -127,20 +127,22 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl } boolean isAnonymous = (clientId == null); LocalDate today = LocalDate.now(); + try { + if (isAnonymous) { + if (!isWhiteListed(ipAddress)) { + LOG.info("ApiRateLimitFilter anonymous request for ip: " + ipAddress); + this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + } - if (isAnonymous ) { - if(!isWhiteListed(ipAddress)) { - LOG.info("ApiRateLimitFilter anonymous request for ip: " + ipAddress); - this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + } else { + LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); + this.rateLimitClientRequest(clientId, today); } - - } else { - LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); - this.rateLimitClientRequest(clientId, today); + } catch (Exception ex) { + LOG.error("Papi Limiting Filter unexpected error, ignore and chain request.", ex); } - - filterChain.doFilter(httpServletRequest, httpServletResponse); } + filterChain.doFilter(httpServletRequest, httpServletResponse); } private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException { @@ -243,7 +245,7 @@ private void sendEmail(String clientId, LocalDate requestDate) { // Send the email boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email, SUBJECT, body, html); if (!mailSent) { - throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); + LOG.error("Failed to send email for papi limits, orcid=" + profile.getId() + " email: " + email); } } @@ -264,33 +266,34 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i }); } - - //gets actual client IP address, using the headers that the proxy server ads + + // gets actual client IP address, using the headers that the proxy server + // ads private String getClientIpAddress(HttpServletRequest request) { String ipAddress = request.getHeader("X-FORWARDED-FOR"); if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { ipAddress = request.getHeader("X-REAL-IP"); } if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { - ipAddress = request.getRemoteAddr(); + ipAddress = request.getRemoteAddr(); } if (ipAddress != null && ipAddress.contains(",")) { ipAddress = ipAddress.split(",")[0].trim(); } return ipAddress; - } - + } + private boolean isWhiteListed(String ipAddress) { List papiIpWhiteList = null; - if(StringUtils.isNotBlank(papiWhiteSpaceSeparatedWhiteList)) { + if (StringUtils.isNotBlank(papiWhiteSpaceSeparatedWhiteList)) { papiIpWhiteList = Arrays.asList(papiWhiteSpaceSeparatedWhiteList.split("\\s")); } - - if(papiIpWhiteList != null) { + + if (papiIpWhiteList != null) { return papiIpWhiteList.contains(ipAddress); - + } - return false; + return false; } } From 8223bda6e603daf8e0e403bdc94cac411ef4306c Mon Sep 17 00:00:00 2001 From: github actions Date: Thu, 7 Nov 2024 20:07:53 +0000 Subject: [PATCH 15/15] v2.67.14 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5231d45e93..c529ed272a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.14 - 2024-11-07 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.13...v2.67.14) + ## v2.67.13 - 2024-11-07 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.12...v2.67.13)