Api limiting http request filter (#7110)

* Implementation of API Rate Limiting Filter * Update development.properties * Bug fixes for slack reporting * Added the properties for test file * Added the tracking to panoply for anonymous/known users that exceeded the rate limit * changed props names * Update ApiRateLimitFilter.java to remove system.out * ApiRateLimitFilter.java log as trace when started * Update orcid-t1-web-context.xml --------- Co-authored-by: Angel Montenegro <[email protected]>
ORCID · Oct 30, 2024 · 739fc89 · 739fc89
1 parent 32f3b3b
commit 739fc89
Show file tree

Hide file tree

Showing 20 changed files with 832 additions and 11 deletions.
diff --git a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java
@@ -53,7 +53,10 @@ public enum Features implements Feature {
     EMAIL_DOMAINS,
 
     @Label("Enable email domains in the UI")
-    EMAIL_DOMAINS_UI;
+    EMAIL_DOMAINS_UI,
+
+    @Label("Enforce rate limiting for public API when disabled the rate monitoring is on. When disabled is the mode is monitoring only.")
+    ENABLE_PAPI_RATE_LIMITING;
 
     public boolean isActive() {
         return FeatureContext.getFeatureManager().isActive(this);

diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl
@@ -0,0 +1,19 @@
+<#import "email_macros.ftl" as emailMacros />
+Dear ${emailName},
+
+This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration:
+
+Client Name: ${clientName}
+Client ID: ${clientId}
+
+Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service
+
+If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations.
+
+To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options. 
+
+Warm Regards, 
+ORCID Support Team 
+https://support.orcid.org
+<@emailMacros.msg "email.common.you_have_received_this_email" />
+<#include "email_footer.ftl"/>
diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl
@@ -0,0 +1,36 @@
+<#import "email_macros.ftl" as emailMacros />
+<#escape x as x?html>
+<!DOCTYPE html>
+<html>
+	<head>	
+		<title>${subject}</title>
+	</head>
+	<body>
+		<div style="padding: 20px; padding-top: 10px; width: 700px; margin: auto;">
+			<img src="https://orcid.org/sites/all/themes/orcid/img/orcid-logo.png" alt="ORCID.org"/>
+		    <hr />
+		    <span style="font-family: arial, helvetica, sans-serif; font-size: 15px; color: #494A4C; font-weight: bold;">Dear ${emailName},</span>
+		    <p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration:</p>
+		    <p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">Client Name: ${clientName}</p>
+			<p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">Client ID: ${clientId}</p>
+			<br/>
+            <p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service.</p>
+            <p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations.</p>
+			<p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options.
+     		<br/>
+            <p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C; white-space: pre;">
+            Warm Regards, 
+			ORCID Support Team 
+			https://support.orcid.org
+			</p>
+
+			<p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C;">
+                <@emailMacros.msg "email.common.you_have_received_this_email" />
+            </p>
+			<p style="font-family: arial,  helvetica, sans-serif;font-size: 15px;color: #494A4C;">
+			   <#include "email_footer_html.ftl"/>
+			</p>
+		 </div>
+	 </body>
+ </html>
+ </#escape>
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java
@@ -0,0 +1,15 @@
+package org.orcid.persistence.dao;
+
+import java.time.LocalDate;
+
+import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity;
+
+public interface PublicApiDailyRateLimitDao extends GenericDao<PublicApiDailyRateLimitEntity, Long> {
+
+    PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate);
+    PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate);
+    int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit);
+    int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit);
+    boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient);
+
+}
diff --git a/...sistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/...sistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java
@@ -0,0 +1,126 @@
+package org.orcid.persistence.dao.impl;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import javax.persistence.Query;
+
+import org.orcid.persistence.dao.PublicApiDailyRateLimitDao;
+import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.transaction.annotation.Transactional;
+
+public class PublicApiDailyRateLimitDaoImpl extends GenericDaoImpl<PublicApiDailyRateLimitEntity, Long> implements PublicApiDailyRateLimitDao {
+    private static final Logger LOG = LoggerFactory.getLogger(PublicApiDailyRateLimitDaoImpl.class);
+
+    public PublicApiDailyRateLimitDaoImpl() {
+        super(PublicApiDailyRateLimitEntity.class);
+    }
+
+    @Override
+    public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) {
+        Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate",
+                PublicApiDailyRateLimitEntity.class);
+        nativeQuery.setParameter("clientId", clientId);
+        nativeQuery.setParameter("requestDate", requestDate);
+        List<PublicApiDailyRateLimitEntity> papiRateList = (List<PublicApiDailyRateLimitEntity>) nativeQuery.getResultList();
+        if (papiRateList != null && papiRateList.size() > 0) {
+            if (papiRateList.size() > 1) {
+                LOG.warn("Found more than one entry for the daily papi rate limiting the client: " + clientId + " and request date: " + requestDate.toString());
+            }
+            return (PublicApiDailyRateLimitEntity) papiRateList.get(0);
+        }
+        return null;
+    }
+
+    @Override
+    public PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate) {
+        String baseQuery = "SELECT * FROM public_api_daily_rate_limit p where p.ip_address=:ipAddress and p.request_date=:requestDate";
+
+        Query nativeQuery = entityManager.createNativeQuery(baseQuery, PublicApiDailyRateLimitEntity.class);
+        nativeQuery.setParameter("ipAddress", ipAddress);
+        nativeQuery.setParameter("requestDate", requestDate);
+
+        List<PublicApiDailyRateLimitEntity> papiRateList = (List<PublicApiDailyRateLimitEntity>) nativeQuery.getResultList();
+        if (papiRateList != null && papiRateList.size() > 0) {
+            LOG.debug("found results ....");
+            if (papiRateList.size() > 1) {
+                LOG.warn("Found more than one entry for the daily papi rate limiting, the IP Address: " + ipAddress + " and request date: " + requestDate.toString());
+            }
+            return (PublicApiDailyRateLimitEntity) papiRateList.get(0);
+        }
+        return null;
+    }
+
+    public int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit) {
+        Query nativeQuery = entityManager.createNativeQuery(
+                "SELECT count(*) FROM public_api_daily_rate_limit p WHERE NOT ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount");
+        nativeQuery.setParameter("requestDate", requestDate);
+        nativeQuery.setParameter("requestCount", limit);
+        List<java.math.BigInteger>  tsList = nativeQuery.getResultList();
+        if (tsList != null && !tsList.isEmpty()) {
+            return tsList.get(0).intValue();
+        }
+        return 0;
+
+    }
+
+    public int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit) {
+        Query nativeQuery = entityManager.createNativeQuery(
+                "SELECT count(*) FROM public_api_daily_rate_limit p WHERE ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount");
+        nativeQuery.setParameter("requestDate", requestDate);
+        nativeQuery.setParameter("requestCount", limit);
+        List<java.math.BigInteger> tsList = nativeQuery.getResultList();
+        if (tsList != null && !tsList.isEmpty()) {
+            return tsList.get(0).intValue();
+        }
+        return 0;
+    }
+
+    @Override
+    @Transactional
+    public boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient) {
+        Query query;
+        if (isClient) {
+            query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where "
+                    + "client_id = :clientId and request_date =:requestDate");
+            query.setParameter("clientId", papiRateLimitingEntity.getClientId());
+        } else {
+            query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where "
+                    + "ip_address = :ipAddress and request_date =:requestDate");
+            query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress());
+        }
+        query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount());
+        query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate().toString());
+        return query.executeUpdate() > 0;
+    }
+
+    @Override
+    @Transactional
+    public void persist(PublicApiDailyRateLimitEntity papiRateLimitingEntity) {
+        String insertQuery = "INSERT INTO public_api_daily_rate_limit " + "(id, client_id, ip_address, request_count, request_date, date_created, last_modified)"
+                + " VALUES ( NEXTVAL('papi_daily_limit_seq'), :clientId , :ipAddress, :requestCount," + " :requestDate, now(), now())";
+
+        Query query = entityManager.createNativeQuery(insertQuery);
+        query.setParameter("clientId", papiRateLimitingEntity.getClientId());
+        query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress());
+        query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount());
+        query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate());
+        query.executeUpdate();
+        return;
+    }
+
+    private static String logQueryWithParams(String baseQuery, Map<String, Object> params) {
+        for (Map.Entry<String, Object> entry : params.entrySet()) {
+            String paramPlaceholder = ":" + entry.getKey();
+            String paramValue = (entry.getValue() instanceof String) ? "'" + entry.getValue() + "'" : entry.getValue().toString();
+            baseQuery = baseQuery.replace(paramPlaceholder, paramValue);
+        }
+        return baseQuery;
+    }
+
+}
diff --git a/...tence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java b/...tence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java
@@ -0,0 +1,129 @@
+package org.orcid.persistence.jpa.entities;
+
+import java.io.Serializable;
+import java.time.LocalDate;
+import java.util.Collection;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.EntityManager;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.PrePersist;
+import javax.persistence.PreUpdate;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+
+
+@Entity
+@Table(name = "public_api_daily_rate_limit")
+public class PublicApiDailyRateLimitEntity implements OrcidEntity<Long>{
+
+    private static final long serialVersionUID = 7137838021634312424L;
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.AUTO, generator = "papi_daily_limit_seq")
+    @SequenceGenerator(name = "papi_daily_limit_seq", sequenceName = "papi_daily_limit_seq", allocationSize = 1)
+    private Long id;
+
+    @Column(name = "client_id", nullable = true)
+    private String clientId;
+
+    @Column(name = "ip_address", nullable = true)
+    private String ipAddress;
+
+    @Column(name = "request_count", nullable = false)
+    private Long requestCount;
+
+    @Column(name = "request_date", nullable = false)
+    private LocalDate requestDate;
+
+    @Column(name = "date_created", nullable = false)
+    private Date dateCreated;
+
+    @Column(name = "last_modified", nullable = false)
+    private Date lastModified;
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Long getId() {
+        return id;
+    }
+
+    public String getClientId() {
+        return clientId;
+    }
+
+    public void setClientId(String clientId) {
+        this.clientId = clientId;
+    }
+
+    public String getIpAddress() {
+        return ipAddress;
+    }
+
+    public void setIpAddress(String ipAddress) {
+        this.ipAddress = ipAddress;
+    }
+
+    public Long getRequestCount() {
+        return requestCount;
+    }
+
+    public void setRequestCount(Long requestCount) {
+        this.requestCount = requestCount;
+    }
+
+    public LocalDate getRequestDate() {
+        return requestDate;
+    }
+
+    public void setRequestDate(LocalDate requestDate) {
+        this.requestDate = requestDate;
+    }
+
+
+    public Date getDateCreated() {
+        return dateCreated;
+    }    
+
+    void setDateCreated(Date date) {
+        this.dateCreated = date;
+    }
+
+    public Date getLastModified() {
+        return lastModified;
+    }
+
+    void setLastModified(Date lastModified) {
+        this.lastModified = lastModified;
+    }
+
+    @PreUpdate
+    void preUpdate() {
+        lastModified = new Date();
+    }
+
+    @PrePersist
+    void prePersist() {
+        Date now = new Date();
+        dateCreated = now;
+        lastModified = now;
+    }
+
+    public static <I extends Serializable, E extends OrcidEntity<I>> Map<I, E> mapById(Collection<E> entities) {
+        Map<I, E> map = new HashMap<I, E>(entities.size());
+        for (E entity : entities) {
+            map.put(entity.getId(), entity);
+        }
+        return map;
+    }
+
+}
+
diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml
@@ -96,6 +96,8 @@
 		<class>org.orcid.statistics.jpa.entities.StatisticValuesEntity</class>
 		<class>org.orcid.statistics.jpa.entities.StatisticKeyEntity</class>
 
+		<!-- PAPI Rate Limitig -->
+		<class>org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity</class>
 		<exclude-unlisted-classes>true</exclude-unlisted-classes>
 
 <!--         <properties> -->

diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml
@@ -405,4 +405,5 @@
   <include file="/db/updates/create_dw_profile_email_domain.xml" />
   <include file="/db/updates/add_unique_constraint_external_id_disambiguated_org.xml" />
   <include file="/db/updates/add_date_verified_to_email.xml" />
+  <include file="/db/updates/create_public_api_daily_rate_limit.xml" />
 </databaseChangeLog>