diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f2df3eacb0..c529ed272a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,37 @@ +## v2.67.14 - 2024-11-07 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.13...v2.67.14) + +## v2.67.13 - 2024-11-07 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.12...v2.67.13) + +## v2.67.12 - 2024-11-06 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.11...v2.67.12) + +## v2.67.11 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.10...v2.67.11) + +## v2.67.10 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.9...v2.67.10) + +## v2.67.9 - 2024-11-05 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.8...v2.67.9) + +## v2.67.8 - 2024-11-04 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.7...v2.67.8) + +## v2.67.7 - 2024-11-04 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.6...v2.67.7) + +- [#7120](https://github.com/ORCID/ORCID-Source/pull/7120): 9422-email-domains-interstitial-oauth + ## v2.67.6 - 2024-10-31 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.5...v2.67.6) diff --git a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java index 7169054c741..1397e77f4e2 100644 --- a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java +++ b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java @@ -25,4 +25,6 @@ public interface OrcidTokenStore extends TokenStore { OAuth2Authentication readAuthenticationFromCachedToken(Map cachedTokenData); void isClientEnabled(String clientId) throws InvalidTokenException; + + String readClientId(String tokenValue); } diff --git a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java index 7d979566bd4..19d50458664 100644 --- a/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java +++ b/orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java @@ -72,6 +72,16 @@ public OrcidOauth2TokenDetail readOrcidOauth2TokenDetail(String token) { return orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token); } + @Override + public String readClientId(String token) { + String clientId = null; + OrcidOauth2TokenDetail orcidTokenDetail = orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token); + if(orcidTokenDetail != null) { + clientId = orcidTokenDetail.getClientDetailsId(); + } + return clientId; + } + /** * Read the authentication stored under the specified token value. * diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl index 0c4345237cd..942550a8898 100644 --- a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl @@ -15,5 +15,5 @@ To minimize any disruption to your ORCID integration in the future, we would rec Warm Regards, ORCID Support Team https://support.orcid.org -<@emailMacros.msg "email.common.you_have_received_this_email" /> + <#include "email_footer.ftl"/> diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl index 569cb86ebe3..f7d4ebca59b 100644 --- a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl @@ -25,9 +25,6 @@

- <@emailMacros.msg "email.common.you_have_received_this_email" /> -

-

<#include "email_footer_html.ftl"/>

diff --git a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml index ac1054dee18..36a8bbddebb 100644 --- a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml +++ b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml @@ -161,13 +161,13 @@ - - - - - - - + + + + + + + diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java index 589811693b4..0955b228b3e 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java @@ -23,7 +23,7 @@ public PublicApiDailyRateLimitDaoImpl() { @Override public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) { - Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate", + Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p where p.client_id=:clientId and p.request_date=:requestDate", PublicApiDailyRateLimitEntity.class); nativeQuery.setParameter("clientId", clientId); nativeQuery.setParameter("requestDate", requestDate); diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 03fce2ff5d2..a96f17593f1 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -460,13 +460,13 @@ - - - - - - - + + + + + + + diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 4a6105494d1..58779274a40 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -3,7 +3,9 @@ import java.io.IOException; import java.io.PrintWriter; import java.time.LocalDate; +import java.util.Arrays; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -14,12 +16,14 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.LocaleUtils; +import org.apache.commons.lang3.StringUtils; import org.orcid.core.manager.ClientDetailsEntityCacheManager; import org.orcid.core.manager.OrcidSecurityManager; import org.orcid.core.manager.TemplateManager; import org.orcid.core.manager.impl.OrcidUrlManager; import org.orcid.core.manager.v3.EmailManager; import org.orcid.core.manager.v3.RecordNameManager; +import org.orcid.core.oauth.service.OrcidTokenStore; import org.orcid.core.utils.OrcidRequestUtil; import org.orcid.persistence.dao.ProfileDao; import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; @@ -33,6 +37,7 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.MessageSource; import org.springframework.http.HttpStatus; import org.springframework.stereotype.Component; import org.springframework.web.filter.OncePerRequestFilter; @@ -69,9 +74,15 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Autowired private EmailManager emailManager; - + @Resource - private PanoplyRedshiftClient panoplyClient; + private PanoplyRedshiftClient panoplyClient; + + @Autowired + private OrcidTokenStore orcidTokenStore; + + @Autowired + private MessageSource messageSource; @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; @@ -81,9 +92,12 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Value("${org.orcid.papi.rate.limit.enabled:false}") private boolean enableRateLimiting; - + @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") - private boolean enablePanoplyPapiExceededRateInProduction; + private boolean enablePanoplyPapiExceededRateInProduction; + + @Value("${org.orcid.papi.rate.limit.ip.whiteSpaceSeparatedWhiteList:127.0.0.1}") + private String papiWhiteSpaceSeparatedWhiteList; private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + "You can increase your daily quota by registering for and using Public API client credentials " @@ -97,22 +111,38 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl throws ServletException, IOException { LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); if (enableRateLimiting) { - String clientId = orcidSecurityManager.getClientIdFromAPIRequest(); - String ipAddress = OrcidRequestUtil.getIpAddress(httpServletRequest); + String tokenValue = null; + if (httpServletRequest.getHeader("Authorization") != null) { + tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim(); + } + String ipAddress = getClientIpAddress(httpServletRequest); + + String clientId = null; + if (tokenValue != null) { + try { + clientId = orcidTokenStore.readClientId(tokenValue); + } catch (Exception ex) { + LOG.error("Exception when trying to get the client id from token value, ignoring and treating as anonymous client", ex); + } + } boolean isAnonymous = (clientId == null); LocalDate today = LocalDate.now(); + try { + if (isAnonymous) { + if (!isWhiteListed(ipAddress)) { + LOG.info("ApiRateLimitFilter anonymous request for ip: " + ipAddress); + this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + } - if (isAnonymous) { - LOG.info("ApiRateLimitFilter anonymous request"); - this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); - - } else { - LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); - this.rateLimitClientRequest(clientId, today); + } else { + LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); + this.rateLimitClientRequest(clientId, today); + } + } catch (Exception ex) { + LOG.error("Papi Limiting Filter unexpected error, ignore and chain request.", ex); } - - filterChain.doFilter(httpServletRequest, httpServletResponse); } + filterChain.doFilter(httpServletRequest, httpServletResponse); } private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException { @@ -163,24 +193,25 @@ private void rateLimitClientRequest(String clientId, LocalDate today) { } // update the request count rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); - papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity,true); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, true); } else { // create rateLimitEntity = new PublicApiDailyRateLimitEntity(); rateLimitEntity.setClientId(clientId); - rateLimitEntity.setRequestCount(0L); + rateLimitEntity.setRequestCount(1L); rateLimitEntity.setRequestDate(today); papiRateLimitingDao.persist(rateLimitEntity); } - } private Map createTemplateParams(String clientId, String clientName, String emailName, String orcidId) { Map templateParams = new HashMap(); + templateParams.put("messages", messageSource); + templateParams.put("messageArgs", new Object[0]); templateParams.put("clientId", clientId); - templateParams.put("clientId", clientName); + templateParams.put("clientName", clientName); templateParams.put("emailName", emailName); templateParams.put("locale", LocaleUtils.toLocale("en")); templateParams.put("baseUri", orcidUrlManager.getBaseUrl()); @@ -195,9 +226,9 @@ private void sendEmail(String clientId, LocalDate requestDate) { String emailName = recordNameManager.deriveEmailFriendlyName(profile.getId()); Map templateParams = this.createTemplateParams(clientId, clientDetailsEntity.getClientName(), emailName, profile.getId()); // Generate body from template - String body = templateManager.processTemplate("bad_orgs_email.ftl", templateParams); + String body = templateManager.processTemplate("papi_rate_limit_email.ftl", templateParams); // Generate html from template - String html = templateManager.processTemplate("bad_orgs_email_html.ftl", templateParams); + String html = templateManager.processTemplate("papi_rate_limit_email_html.ftl", templateParams); String email = emailManager.findPrimaryEmail(profile.getId()).getEmail(); LOG.info("text email={}", body); @@ -212,15 +243,14 @@ private void sendEmail(String clientId, LocalDate requestDate) { } // Send the email - boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html); + boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email, SUBJECT, body, html); if (!mailSent) { - throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); + LOG.error("Failed to send email for papi limits, orcid=" + profile.getId() + " email: " + email); } } - - + private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) { - //Store the rate exceeded item in panoply Db without blocking + // Store the rate exceeded item in panoply Db without blocking CompletableFuture.supplyAsync(() -> { try { panoplyClient.addPanoplyPapiDailyRateExceeded(item); @@ -229,12 +259,41 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i LOG.error("Cannot store the rateExceededItem to panoply ", e); return false; } - }).thenAccept(result -> { - if(! result) { - LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + }).thenAccept(result -> { + if (!result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result); } }); } + // gets actual client IP address, using the headers that the proxy server + // ads + private String getClientIpAddress(HttpServletRequest request) { + String ipAddress = request.getHeader("X-FORWARDED-FOR"); + if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { + ipAddress = request.getHeader("X-REAL-IP"); + } + if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { + ipAddress = request.getRemoteAddr(); + } + if (ipAddress != null && ipAddress.contains(",")) { + ipAddress = ipAddress.split(",")[0].trim(); + } + return ipAddress; + } + + private boolean isWhiteListed(String ipAddress) { + List papiIpWhiteList = null; + if (StringUtils.isNotBlank(papiWhiteSpaceSeparatedWhiteList)) { + papiIpWhiteList = Arrays.asList(papiWhiteSpaceSeparatedWhiteList.split("\\s")); + } + + if (papiIpWhiteList != null) { + return papiIpWhiteList.contains(ipAddress); + + } + return false; + } + } diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java index 0e63e4f9054..73b1b4258c1 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -37,13 +37,13 @@ public class PapiDailyLimitReport { @Value("${org.orcid.core.orgs.load.slackUser}") private String slackUser; - @Value("${rate.limit.anonymous.requests}") + @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") private int anonymousRequestLimit; - @Value("${rate.limit.known.requests}") + @Value("${org.orcid.papi.rate.limit.known.requests:40000}") private int knownRequestLimit; - @Value("${rate.limit.enabled:false}") + @Value("${org.orcid.papi.rate.limit.enabled:false}") private boolean enableRateLimiting; @Autowired @@ -71,13 +71,13 @@ public void papiDailyLimitReport() { if (enableRateLimiting) { LocalDate yesterday = LocalDate.now().minusDays(1); String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; - String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\n Current Anonymous Requests Limit: " + anonymousRequestLimit - + "\n Current Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; + String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\nCurrent Anonymous Requests Limit: " + anonymousRequestLimit + + "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\nMode: " + mode; LOG .info(SLACK_INTRO_MSG); slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl); String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit) - + "\n Count of Public API clients that have exceeded the limit: " + + "\nCount of Public API clients that have exceeded the limit: " + papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit); LOG .info(SLACK_STATS_MSG); slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl); diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java index 72997bfb7bf..9c12d1c02f1 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java @@ -535,12 +535,14 @@ public ModelAndView confirmDeactivateOrcidAccount(HttpServletRequest request, Ht } org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(v2Emails, emailDomains); // Old emails are missing the source name and id -- assign the user as the source - for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { - if (email.getSource() == null && email.getSourceName() == null) { - String orcid = getCurrentUserOrcid(); - String displayName = getPersonDetails(orcid, true).getDisplayName(); - email.setSource(orcid); - email.setSourceName(displayName); + if (emails.getEmails() != null) { + for (org.orcid.pojo.ajaxForm.Email email : emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + String orcid = getCurrentUserOrcid(); + String displayName = getPersonDetails(orcid, true).getDisplayName(); + email.setSource(orcid); + email.setSourceName(displayName); + } } } return emails; diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java index b0787513446..9de6f1207fe 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java @@ -205,12 +205,15 @@ PublicRecord getRecord(String orcid) { org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(filteredEmails, emailDomains); // Old emails are missing the source name and id -- assign the user as the source - for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { - if (email.getSource() == null && email.getSourceName() == null) { - email.setSource(orcid); - email.setSourceName(publicRecord.getDisplayName()); + if (emails.getEmails() != null) { + for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + email.setSource(orcid); + email.setSourceName(publicRecord.getDisplayName()); + } } } + publicRecord.setEmails(emails); // Fill external identifiers