diff --git a/CHANGELOG.md b/CHANGELOG.md index 6665393febc..cceb5e9446e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## v2.67.2 - 2024-10-30 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.1...v2.67.2) + +## v2.67.1 - 2024-10-29 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.0...v2.67.1) + +## v2.67.0 - 2024-10-29 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.6...v2.67.0) + ## v2.66.6 - 2024-10-28 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.5...v2.66.6) diff --git a/orcid-activemq/src/main/resources/message-listener.properties b/orcid-activemq/src/main/resources/message-listener.properties index f7ff7385fce..f4d5343a573 100644 --- a/orcid-activemq/src/main/resources/message-listener.properties +++ b/orcid-activemq/src/main/resources/message-listener.properties @@ -70,4 +70,15 @@ org.orcid.message-listener.index.s3.search.max_elements=5 org.orcid.message-listener.index.summaries.bucket_name=v2.0-summaries-dev org.orcid.message-listener.index.activities.bucket_name=v2.0-activities-dev org.orcid.message-listener.index.summaries.v3.bucket_name=v3.0-summaries-dev -org.orcid.message-listener.index.activities.v3.bucket_name=v3.0-activities-dev \ No newline at end of file +org.orcid.message-listener.index.activities.v3.bucket_name=v3.0-activities-dev + + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java index 389a28d2dc9..d975b43664c 100644 --- a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java +++ b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java @@ -53,7 +53,10 @@ public enum Features implements Feature { EMAIL_DOMAINS, @Label("Enable email domains in the UI") - EMAIL_DOMAINS_UI; + EMAIL_DOMAINS_UI, + + @Label("Enforce rate limiting for public API when disabled the rate monitoring is on. When disabled is the mode is monitoring only.") + ENABLE_PAPI_RATE_LIMITING; public boolean isActive() { return FeatureContext.getFeatureManager().isActive(this); diff --git a/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java b/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java index cb396d94486..2e3528fcbdd 100644 --- a/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java +++ b/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java @@ -16,45 +16,29 @@ import org.springframework.web.filter.OncePerRequestFilter; /** - * * @author Robert Peters (rcpeters) - * */ public class CorsFilterWeb extends OncePerRequestFilter { @Resource CrossDomainWebManger crossDomainWebManger; - - private static final String LOCALHOST_BASE_URI= "https://localhost"; - private static final String LOCALHOST_ORCID_WEB_BASE_URI = "https://localhost:8443/orcid-web"; - + @Value("${org.orcid.core.baseUri}") private String baseUri; @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { - Pattern p = Pattern.compile("^/userStatus\\.json|^/oauth/userinfo|^/oauth/jwks|^/\\.well-known/openid-configuration"); - Matcher m = p.matcher(OrcidUrlManager.getPathWithoutContextPath(request)); - // Allow CORS for all paths from Angular frontend only if we are in local dev env - // All other envs allow CORS only if request path matches one of: - // userStatus.json - // /oauth/userinfo - // /oauth/jwks - // /.well-known/openid-configuration - if (baseUri.equals(LOCALHOST_BASE_URI) || baseUri.equals(LOCALHOST_ORCID_WEB_BASE_URI) || m.matches()) { - if (crossDomainWebManger.allowed(request)) { - String origin = request.getHeader("origin"); - response.addHeader("Access-Control-Allow-Origin", origin); - response.addHeader("Access-Control-Allow-Credentials", "true"); - - if (request.getHeader("Access-Control-Request-Method") != null && "OPTIONS".equals(request.getMethod())) { - // CORS "pre-flight" request - response.addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE"); - response.addHeader("Access-Control-Allow-Headers", "X-Requested-With,Origin,Content-Type,Accept,Authorization,x-csrf-token"); - } + if (crossDomainWebManger.allowed(request)) { + String origin = request.getHeader("origin"); + response.addHeader("Access-Control-Allow-Origin", origin); + response.addHeader("Access-Control-Allow-Credentials", "true"); + + if (request.getHeader("Access-Control-Request-Method") != null && "OPTIONS".equals(request.getMethod())) { + // CORS "pre-flight" request + response.addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE"); + response.addHeader("Access-Control-Allow-Headers", "X-Requested-With,Origin,Content-Type,Accept,Authorization,x-csrf-token"); } - } filterChain.doFilter(request, response); diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl new file mode 100644 index 00000000000..0c4345237cd --- /dev/null +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl @@ -0,0 +1,19 @@ +<#import "email_macros.ftl" as emailMacros /> +Dear ${emailName}, + +This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration: + +Client Name: ${clientName} +Client ID: ${clientId} + +Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service + +If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations. + +To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options. + +Warm Regards, +ORCID Support Team +https://support.orcid.org +<@emailMacros.msg "email.common.you_have_received_this_email" /> +<#include "email_footer.ftl"/> diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl new file mode 100644 index 00000000000..569cb86ebe3 --- /dev/null +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl @@ -0,0 +1,36 @@ +<#import "email_macros.ftl" as emailMacros /> +<#escape x as x?html> + + + + ${subject} + + +
+ ORCID.org +
+ Dear ${emailName}, +

This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration:

+

Client Name: ${clientName}

+

Client ID: ${clientId}

+
+

Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service.

+

If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations.

+

To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options. +
+

+ Warm Regards, + ORCID Support Team + https://support.orcid.org +

+ +

+ <@emailMacros.msg "email.common.you_have_received_this_email" /> +

+

+ <#include "email_footer_html.ftl"/> +

+
+ + + diff --git a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml index 13166f997da..ac1054dee18 100644 --- a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml +++ b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml @@ -158,4 +158,24 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java new file mode 100644 index 00000000000..6d090690792 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java @@ -0,0 +1,15 @@ +package org.orcid.persistence.dao; + +import java.time.LocalDate; + +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; + +public interface PublicApiDailyRateLimitDao extends GenericDao { + + PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate); + PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate); + int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit); + int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit); + boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient); + +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java new file mode 100644 index 00000000000..75f71549255 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java @@ -0,0 +1,126 @@ +package org.orcid.persistence.dao.impl; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.persistence.Query; + +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; + +public class PublicApiDailyRateLimitDaoImpl extends GenericDaoImpl implements PublicApiDailyRateLimitDao { + private static final Logger LOG = LoggerFactory.getLogger(PublicApiDailyRateLimitDaoImpl.class); + + public PublicApiDailyRateLimitDaoImpl() { + super(PublicApiDailyRateLimitEntity.class); + } + + @Override + public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) { + Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate", + PublicApiDailyRateLimitEntity.class); + nativeQuery.setParameter("clientId", clientId); + nativeQuery.setParameter("requestDate", requestDate); + List papiRateList = (List) nativeQuery.getResultList(); + if (papiRateList != null && papiRateList.size() > 0) { + if (papiRateList.size() > 1) { + LOG.warn("Found more than one entry for the daily papi rate limiting the client: " + clientId + " and request date: " + requestDate.toString()); + } + return (PublicApiDailyRateLimitEntity) papiRateList.get(0); + } + return null; + } + + @Override + public PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate) { + String baseQuery = "SELECT * FROM public_api_daily_rate_limit p where p.ip_address=:ipAddress and p.request_date=:requestDate"; + + Query nativeQuery = entityManager.createNativeQuery(baseQuery, PublicApiDailyRateLimitEntity.class); + nativeQuery.setParameter("ipAddress", ipAddress); + nativeQuery.setParameter("requestDate", requestDate); + + List papiRateList = (List) nativeQuery.getResultList(); + if (papiRateList != null && papiRateList.size() > 0) { + LOG.debug("found results ...."); + if (papiRateList.size() > 1) { + LOG.warn("Found more than one entry for the daily papi rate limiting, the IP Address: " + ipAddress + " and request date: " + requestDate.toString()); + } + return (PublicApiDailyRateLimitEntity) papiRateList.get(0); + } + return null; + } + + public int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit) { + Query nativeQuery = entityManager.createNativeQuery( + "SELECT count(*) FROM public_api_daily_rate_limit p WHERE NOT ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount"); + nativeQuery.setParameter("requestDate", requestDate); + nativeQuery.setParameter("requestCount", limit); + List tsList = nativeQuery.getResultList(); + if (tsList != null && !tsList.isEmpty()) { + return tsList.get(0).intValue(); + } + return 0; + + } + + public int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit) { + Query nativeQuery = entityManager.createNativeQuery( + "SELECT count(*) FROM public_api_daily_rate_limit p WHERE ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount"); + nativeQuery.setParameter("requestDate", requestDate); + nativeQuery.setParameter("requestCount", limit); + List tsList = nativeQuery.getResultList(); + if (tsList != null && !tsList.isEmpty()) { + return tsList.get(0).intValue(); + } + return 0; + } + + @Override + @Transactional + public boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient) { + Query query; + if (isClient) { + query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where " + + "client_id = :clientId and request_date =:requestDate"); + query.setParameter("clientId", papiRateLimitingEntity.getClientId()); + } else { + query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where " + + "ip_address = :ipAddress and request_date =:requestDate"); + query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress()); + } + query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount()); + query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate().toString()); + return query.executeUpdate() > 0; + } + + @Override + @Transactional + public void persist(PublicApiDailyRateLimitEntity papiRateLimitingEntity) { + String insertQuery = "INSERT INTO public_api_daily_rate_limit " + "(id, client_id, ip_address, request_count, request_date, date_created, last_modified)" + + " VALUES ( NEXTVAL('papi_daily_limit_seq'), :clientId , :ipAddress, :requestCount," + " :requestDate, now(), now())"; + + Query query = entityManager.createNativeQuery(insertQuery); + query.setParameter("clientId", papiRateLimitingEntity.getClientId()); + query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress()); + query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount()); + query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate()); + query.executeUpdate(); + return; + } + + private static String logQueryWithParams(String baseQuery, Map params) { + for (Map.Entry entry : params.entrySet()) { + String paramPlaceholder = ":" + entry.getKey(); + String paramValue = (entry.getValue() instanceof String) ? "'" + entry.getValue() + "'" : entry.getValue().toString(); + baseQuery = baseQuery.replace(paramPlaceholder, paramValue); + } + return baseQuery; + } + +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java new file mode 100644 index 00000000000..bc8a9d42ec8 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java @@ -0,0 +1,129 @@ +package org.orcid.persistence.jpa.entities; + +import java.io.Serializable; +import java.time.LocalDate; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EntityManager; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; + + +@Entity +@Table(name = "public_api_daily_rate_limit") +public class PublicApiDailyRateLimitEntity implements OrcidEntity{ + + private static final long serialVersionUID = 7137838021634312424L; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO, generator = "papi_daily_limit_seq") + @SequenceGenerator(name = "papi_daily_limit_seq", sequenceName = "papi_daily_limit_seq", allocationSize = 1) + private Long id; + + @Column(name = "client_id", nullable = true) + private String clientId; + + @Column(name = "ip_address", nullable = true) + private String ipAddress; + + @Column(name = "request_count", nullable = false) + private Long requestCount; + + @Column(name = "request_date", nullable = false) + private LocalDate requestDate; + + @Column(name = "date_created", nullable = false) + private Date dateCreated; + + @Column(name = "last_modified", nullable = false) + private Date lastModified; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getClientId() { + return clientId; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public Long getRequestCount() { + return requestCount; + } + + public void setRequestCount(Long requestCount) { + this.requestCount = requestCount; + } + + public LocalDate getRequestDate() { + return requestDate; + } + + public void setRequestDate(LocalDate requestDate) { + this.requestDate = requestDate; + } + + + public Date getDateCreated() { + return dateCreated; + } + + void setDateCreated(Date date) { + this.dateCreated = date; + } + + public Date getLastModified() { + return lastModified; + } + + void setLastModified(Date lastModified) { + this.lastModified = lastModified; + } + + @PreUpdate + void preUpdate() { + lastModified = new Date(); + } + + @PrePersist + void prePersist() { + Date now = new Date(); + dateCreated = now; + lastModified = now; + } + + public static > Map mapById(Collection entities) { + Map map = new HashMap(entities.size()); + for (E entity : entities) { + map.put(entity.getId(), entity); + } + return map; + } + +} + diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml index e92cdc61d71..dda9237067f 100644 --- a/orcid-persistence/src/main/resources/META-INF/persistence.xml +++ b/orcid-persistence/src/main/resources/META-INF/persistence.xml @@ -96,6 +96,8 @@ org.orcid.statistics.jpa.entities.StatisticValuesEntity org.orcid.statistics.jpa.entities.StatisticKeyEntity + + org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity true diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index 65dfe060bf5..a4a3b726fdb 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -405,4 +405,5 @@ + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml b/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml new file mode 100644 index 00000000000..1b575f020dd --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SELECT 1 FROM pg_roles WHERE rolname='orcidro' + + GRANT SELECT ON public_api_daily_rate_limit TO orcidro; + + + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 629b3b163c1..03fce2ff5d2 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -482,4 +482,8 @@ + + + + diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java new file mode 100644 index 00000000000..4a6105494d1 --- /dev/null +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -0,0 +1,240 @@ +package org.orcid.api.filters; + +import java.io.IOException; +import java.io.PrintWriter; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import javax.annotation.Resource; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang.LocaleUtils; +import org.orcid.core.manager.ClientDetailsEntityCacheManager; +import org.orcid.core.manager.OrcidSecurityManager; +import org.orcid.core.manager.TemplateManager; +import org.orcid.core.manager.impl.OrcidUrlManager; +import org.orcid.core.manager.v3.EmailManager; +import org.orcid.core.manager.v3.RecordNameManager; +import org.orcid.core.utils.OrcidRequestUtil; +import org.orcid.persistence.dao.ProfileDao; +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; +import org.orcid.persistence.jpa.entities.ClientDetailsEntity; +import org.orcid.persistence.jpa.entities.ProfileEntity; +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; +import org.orcid.utils.email.MailGunManager; +import org.orcid.utils.panoply.PanoplyPapiDailyRateExceededItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import org.orcid.core.togglz.Features; + +@Component +public class ApiRateLimitFilter extends OncePerRequestFilter { + private static Logger LOG = LoggerFactory.getLogger(ApiRateLimitFilter.class); + + @Autowired + private PublicApiDailyRateLimitDao papiRateLimitingDao; + + @Autowired + private OrcidSecurityManager orcidSecurityManager; + + @Autowired + private ClientDetailsEntityCacheManager clientDetailsEntityCacheManager; + + @Autowired + private MailGunManager mailGunManager; + + @Autowired + private ProfileDao profileDao; + + @Autowired + private OrcidUrlManager orcidUrlManager; + + @Autowired + private RecordNameManager recordNameManager; + + @Autowired + private TemplateManager templateManager; + + @Autowired + private EmailManager emailManager; + + @Resource + private PanoplyRedshiftClient panoplyClient; + + @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") + private int anonymousRequestLimit; + + @Value("${org.orcid.papi.rate.limit.known.requests:40000}") + private int knownRequestLimit; + + @Value("${org.orcid.papi.rate.limit.enabled:false}") + private boolean enableRateLimiting; + + @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") + private boolean enablePanoplyPapiExceededRateInProduction; + + private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + + "You can increase your daily quota by registering for and using Public API client credentials " + + "(https://info.orcid.org/documentation/integration-guide/registering-a-public-api-client/ )"; + + private static final String SUBJECT = "[ORCID] You have exceeded the daily Public API Usage Limit - "; + private static final String FROM_ADDRESS = "\"Engagement Team, ORCID\" "; + + @Override + protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, FilterChain filterChain) + throws ServletException, IOException { + LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); + if (enableRateLimiting) { + String clientId = orcidSecurityManager.getClientIdFromAPIRequest(); + String ipAddress = OrcidRequestUtil.getIpAddress(httpServletRequest); + boolean isAnonymous = (clientId == null); + LocalDate today = LocalDate.now(); + + if (isAnonymous) { + LOG.info("ApiRateLimitFilter anonymous request"); + this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + + } else { + LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); + this.rateLimitClientRequest(clientId, today); + } + + filterChain.doFilter(httpServletRequest, httpServletResponse); + } + } + + private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException { + PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByIpAddressAndRequestDate(ipAddress, today); + if (rateLimitEntity != null) { + // update the request count only when limit not exceeded ? + rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, false); + if (rateLimitEntity.getRequestCount() == knownRequestLimit && enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setIpAddress(ipAddress); + item.setRequestDate(rateLimitEntity.getRequestDate()); + setPapiRateExceededItemInPanoply(item); + } + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { + if (rateLimitEntity.getRequestCount() >= anonymousRequestLimit) { + httpServletResponse.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + if (!httpServletResponse.isCommitted()) { + try (PrintWriter writer = httpServletResponse.getWriter()) { + writer.write(TOO_MANY_REQUESTS_MSG); + writer.flush(); + } + return; + } + } + } + } else { + // create + rateLimitEntity = new PublicApiDailyRateLimitEntity(); + rateLimitEntity.setIpAddress(ipAddress); + rateLimitEntity.setRequestCount(1L); + rateLimitEntity.setRequestDate(today); + papiRateLimitingDao.persist(rateLimitEntity); + + } + return; + + } + + private void rateLimitClientRequest(String clientId, LocalDate today) { + PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByClientIdAndRequestDate(clientId, today); + if (rateLimitEntity != null) { + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { + // email the client first time the limit is reached + if (rateLimitEntity.getRequestCount() == knownRequestLimit) { + sendEmail(clientId, rateLimitEntity.getRequestDate()); + } + } + // update the request count + rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity,true); + + } else { + // create + rateLimitEntity = new PublicApiDailyRateLimitEntity(); + rateLimitEntity.setClientId(clientId); + rateLimitEntity.setRequestCount(0L); + rateLimitEntity.setRequestDate(today); + papiRateLimitingDao.persist(rateLimitEntity); + } + + + } + + private Map createTemplateParams(String clientId, String clientName, String emailName, String orcidId) { + Map templateParams = new HashMap(); + templateParams.put("clientId", clientId); + templateParams.put("clientId", clientName); + templateParams.put("emailName", emailName); + templateParams.put("locale", LocaleUtils.toLocale("en")); + templateParams.put("baseUri", orcidUrlManager.getBaseUrl()); + templateParams.put("baseUriHttp", orcidUrlManager.getBaseUriHttp()); + templateParams.put("subject", SUBJECT + orcidId); + return templateParams; + } + + private void sendEmail(String clientId, LocalDate requestDate) { + ClientDetailsEntity clientDetailsEntity = clientDetailsEntityCacheManager.retrieve(clientId); + ProfileEntity profile = profileDao.find(clientDetailsEntity.getGroupProfileId()); + String emailName = recordNameManager.deriveEmailFriendlyName(profile.getId()); + Map templateParams = this.createTemplateParams(clientId, clientDetailsEntity.getClientName(), emailName, profile.getId()); + // Generate body from template + String body = templateManager.processTemplate("bad_orgs_email.ftl", templateParams); + // Generate html from template + String html = templateManager.processTemplate("bad_orgs_email_html.ftl", templateParams); + String email = emailManager.findPrimaryEmail(profile.getId()).getEmail(); + + LOG.info("text email={}", body); + LOG.info("html email={}", html); + if (enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setClientId(clientId); + item.setOrcid(profile.getId()); + item.setEmail(email); + item.setRequestDate(requestDate); + setPapiRateExceededItemInPanoply(item); + } + + // Send the email + boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html); + if (!mailSent) { + throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); + } + } + + + private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) { + //Store the rate exceeded item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyPapiDailyRateExceeded(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store the rateExceededItem to panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); + } + +} diff --git a/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml b/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml index 87ebab08caf..73fe82eef52 100644 --- a/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml +++ b/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml @@ -82,12 +82,13 @@ - - - + + \ No newline at end of file diff --git a/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml b/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml index 60a2db946f8..4189a06e33c 100644 --- a/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml +++ b/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml @@ -16,16 +16,18 @@ - + - + - - - + + + + + @@ -86,4 +88,6 @@ + + diff --git a/orcid-pub-web/src/main/webapp/WEB-INF/web.xml b/orcid-pub-web/src/main/webapp/WEB-INF/web.xml index 1688d337a90..aded7e07657 100644 --- a/orcid-pub-web/src/main/webapp/WEB-INF/web.xml +++ b/orcid-pub-web/src/main/webapp/WEB-INF/web.xml @@ -135,6 +135,16 @@ clientCredentialsTokenEndpointFilter /oauth/token + + + rateLimitingFilter + org.springframework.web.filter.DelegatingFilterProxy + + + + rateLimitingFilter + * + springSecurityFilterChain diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java new file mode 100644 index 00000000000..0e63e4f9054 --- /dev/null +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -0,0 +1,100 @@ +package org.orcid.scheduler.report; + +import java.time.LocalDate; + +import javax.annotation.Resource; + +import org.orcid.core.togglz.Features; +import org.orcid.core.togglz.OrcidTogglzConfiguration; +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; + +import org.orcid.utils.alerting.SlackManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.stereotype.Service; +import org.togglz.core.context.ContextClassLoaderFeatureManagerProvider; +import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.FeatureManagerBuilder; + +@Service +public class PapiDailyLimitReport { + + private static final Logger LOG = LoggerFactory.getLogger(PapiDailyLimitReport.class); + + @Resource + private SlackManager slackManager; + + @Value("${org.orcid.core.papiLimitReport.slackChannel:collab-spam-reports}") + private String slackChannel; + + @Value("${org.orcid.core.papiLimitReport.webhookUrl}") + private String webhookUrl; + + @Value("${org.orcid.core.orgs.load.slackUser}") + private String slackUser; + + @Value("${rate.limit.anonymous.requests}") + private int anonymousRequestLimit; + + @Value("${rate.limit.known.requests}") + private int knownRequestLimit; + + @Value("${rate.limit.enabled:false}") + private boolean enableRateLimiting; + + @Autowired + private PublicApiDailyRateLimitDao papiRateLimitingDao; + + // for running spam manually + public static void main(String[] args) { + PapiDailyLimitReport dailyLimitReport = new PapiDailyLimitReport(); + try { + dailyLimitReport.init(); + dailyLimitReport.papiDailyLimitReport(); + } catch (Exception e) { + LOG.error("Exception when getting the report for daily limit", e); + System.err.println(e.getMessage()); + } finally { + System.exit(0); + } + + } + /** + * Sends daily slack reports to dedicated slack channel. + */ + public void papiDailyLimitReport() { + LOG .info("start papi limit report the rate limiting is: " + enableRateLimiting); + if (enableRateLimiting) { + LocalDate yesterday = LocalDate.now().minusDays(1); + String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; + String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\n Current Anonymous Requests Limit: " + anonymousRequestLimit + + "\n Current Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; + LOG .info(SLACK_INTRO_MSG); + slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl); + + String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit) + + "\n Count of Public API clients that have exceeded the limit: " + + papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit); + LOG .info(SLACK_STATS_MSG); + slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl); + } + + } + + @SuppressWarnings("resource") + private void init() { + ApplicationContext context = new ClassPathXmlApplicationContext("orcid-scheduler-context.xml"); + papiRateLimitingDao = (PublicApiDailyRateLimitDao) context.getBean("papiRateLimitingDao"); + bootstrapTogglz(context.getBean(OrcidTogglzConfiguration.class)); + } + + private static void bootstrapTogglz(OrcidTogglzConfiguration togglzConfig) { + FeatureManager featureManager = new FeatureManagerBuilder().togglzConfig(togglzConfig).build(); + ContextClassLoaderFeatureManagerProvider.bind(featureManager); + } + +} diff --git a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml index 897d1e1a90b..dc6dd87b421 100644 --- a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml +++ b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml @@ -45,6 +45,7 @@ + @@ -138,5 +139,7 @@ + + diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties index f1d06636692..796e8862bad 100644 --- a/orcid-test/src/main/resources/properties/test-scheduler.properties +++ b/orcid-test/src/main/resources/properties/test-scheduler.properties @@ -54,3 +54,11 @@ org.orcid.core.utils.panoply.idleConnectionTimeout=3600 org.orcid.core.utils.panoply.connectionTimeout=36000 org.orcid.core.utils.panoply.jdbcUrl=xxx org.orcid.core.utils.panoply.username=xxx + +#Slack channel for papi limits +org.orcid.core.papiLimitReport.slackChannel=xxx +org.orcid.core.papiLimitReport.webhookUrl=xxx +rate.limit.anonymous.requests=1 +rate.limit.known.requests=2 +rate.limit.enabled=true +org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * \ No newline at end of file diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java new file mode 100644 index 00000000000..9761ed95f43 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java @@ -0,0 +1,49 @@ +package org.orcid.utils.panoply; + +import java.time.LocalDate; + +public class PanoplyPapiDailyRateExceededItem { + private String ipAddress; + private String clientId; + private String email; + private String orcid; + private LocalDate requestDate; + + + public String getIpAddress() { + return ipAddress; + } + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + public String getClientId() { + return clientId; + } + public void setClientId(String clientId) { + this.clientId = clientId; + } + public String getEmail() { + return email; + } + public void setEmail(String email) { + this.email = email; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + public LocalDate getRequestDate() { + return requestDate; + } + public void setRequestDate(LocalDate requestDate) { + this.requestDate = requestDate; + } + + @Override + public String toString() { + return "PanoplyPapiDailyRateExceededItem{" + "ipAddress=" + ipAddress + ", clientId='" + clientId + '\'' + ", email='" + email + '\'' + ", orcid='" + orcid + '\'' + + ", requestDate='" + requestDate + '\'' + '}'; + } +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java index 77c5db8ef16..a7b0d24002f 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -31,5 +31,11 @@ public int addPanoplyDeletedItem(PanoplyDeletedItem item) { return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); } - + + public int addPanoplyPapiDailyRateExceeded(PanoplyPapiDailyRateExceededItem item) { + LOG.debug("Adding papi daily rate exceeded item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_papi_daily_rate_exceeded (ip_address, orcid, client_id, email, request_date) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getIpAddress(), item.getOrcid(), item.getClientId(), item.getEmail(), item.getRequestDate()); + } + } diff --git a/properties/development.properties b/properties/development.properties index 9186b22ceb8..036dd07a4ae 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -99,7 +99,7 @@ org.orcid.core.node = 1 org.orcid.core.numberOfNodes = 1 # CORS allowed domains -org.orcid.security.cors.allowed_domains=localhost +org.orcid.security.cors.allowed_domains=dev.orcid.org # Messaging # Replace with tcp://domain.com:61616 in live to point at ActiveMQ location @@ -258,15 +258,26 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -org.orcid.persistence.liquibase.enabled=false +#org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr - org.orcid.persistence.panoply.cleanup.production=false +#api_limiting_http_request_filter +org.orcid.persistence.panoply.papiExceededRate.production=false + # Panoply redshift database org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver org.orcid.core.utils.panoply.maxPoolSize=20 org.orcid.core.utils.panoply.password=xxx org.orcid.core.utils.panoply.idleConnectionTimeout=3600 org.orcid.core.utils.panoply.connectionTimeout=36000 +#jdbc:redshift://:/ org.orcid.core.utils.panoply.jdbcUrl=xxx org.orcid.core.utils.panoply.username=xxx + +#Slack channel for papi limits +org.orcid.core.papiLimitReport.slackChannel=system-alerts-qa +org.orcid.core.papiLimitReport.webhookUrl=xxx +org.orcid.papi.rate.limit.anonymous.requests=1 +org.orcid.papi.rate.limit.known.requests=2 +org.orcid.papi.rate.limit.enabled=true +org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * \ No newline at end of file