From 7dae1f1410c951817a5dde2541a124432d642270 Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 29 Oct 2024 15:28:56 +0000 Subject: [PATCH 01/12] v2.67.0 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6665393feb..01a2af9c7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.0 - 2024-10-29 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.6...v2.67.0) + ## v2.66.6 - 2024-10-28 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.5...v2.66.6) From e3eddca0c99c06fbb77d5ed00ef46cf5b2599c25 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Tue, 29 Oct 2024 15:29:57 +0000 Subject: [PATCH 02/12] Fixes for the ML startup error (#7114) --- .../resources/message-listener.properties | 13 +++++++++++- .../orcid-message-listener-web-context.xml | 20 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/orcid-activemq/src/main/resources/message-listener.properties b/orcid-activemq/src/main/resources/message-listener.properties index f7ff7385fc..f4d5343a57 100644 --- a/orcid-activemq/src/main/resources/message-listener.properties +++ b/orcid-activemq/src/main/resources/message-listener.properties @@ -70,4 +70,15 @@ org.orcid.message-listener.index.s3.search.max_elements=5 org.orcid.message-listener.index.summaries.bucket_name=v2.0-summaries-dev org.orcid.message-listener.index.activities.bucket_name=v2.0-activities-dev org.orcid.message-listener.index.summaries.v3.bucket_name=v3.0-summaries-dev -org.orcid.message-listener.index.activities.v3.bucket_name=v3.0-activities-dev \ No newline at end of file +org.orcid.message-listener.index.activities.v3.bucket_name=v3.0-activities-dev + + +org.orcid.persistence.panoply.cleanup.production=false +# Panoply redshift database +org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver +org.orcid.core.utils.panoply.maxPoolSize=20 +org.orcid.core.utils.panoply.password=xxx +org.orcid.core.utils.panoply.idleConnectionTimeout=3600 +org.orcid.core.utils.panoply.connectionTimeout=36000 +org.orcid.core.utils.panoply.jdbcUrl=xxx +org.orcid.core.utils.panoply.username=xxx \ No newline at end of file diff --git a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml index 13166f997d..ac1054dee1 100644 --- a/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml +++ b/orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml @@ -158,4 +158,24 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From e68bfbeaa983e0d83286bbc37c46f6e5ad703b0a Mon Sep 17 00:00:00 2001 From: github actions Date: Tue, 29 Oct 2024 15:45:17 +0000 Subject: [PATCH 03/12] v2.67.1 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 01a2af9c7f..5ca3420c95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.1 - 2024-10-29 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.0...v2.67.1) + ## v2.67.0 - 2024-10-29 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.66.6...v2.67.0) From b81578f918c75e96a69e41b5d02528f6d10f6679 Mon Sep 17 00:00:00 2001 From: bobcaprice Date: Wed, 30 Oct 2024 12:58:32 +0000 Subject: [PATCH 04/12] removed endpoint regex from cors filter (#7108) Co-authored-by: George Nash Co-authored-by: Angel Montenegro --- .../orcid/core/web/filters/CorsFilterWeb.java | 36 ++++++------------- properties/development.properties | 4 +-- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java b/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java index cb396d9448..2e3528fcbd 100644 --- a/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java +++ b/orcid-core/src/main/java/org/orcid/core/web/filters/CorsFilterWeb.java @@ -16,45 +16,29 @@ import org.springframework.web.filter.OncePerRequestFilter; /** - * * @author Robert Peters (rcpeters) - * */ public class CorsFilterWeb extends OncePerRequestFilter { @Resource CrossDomainWebManger crossDomainWebManger; - - private static final String LOCALHOST_BASE_URI= "https://localhost"; - private static final String LOCALHOST_ORCID_WEB_BASE_URI = "https://localhost:8443/orcid-web"; - + @Value("${org.orcid.core.baseUri}") private String baseUri; @Override protected void doFilterInternal(HttpServletRequest request, HttpServletResponse response, FilterChain filterChain) throws ServletException, IOException { - Pattern p = Pattern.compile("^/userStatus\\.json|^/oauth/userinfo|^/oauth/jwks|^/\\.well-known/openid-configuration"); - Matcher m = p.matcher(OrcidUrlManager.getPathWithoutContextPath(request)); - // Allow CORS for all paths from Angular frontend only if we are in local dev env - // All other envs allow CORS only if request path matches one of: - // userStatus.json - // /oauth/userinfo - // /oauth/jwks - // /.well-known/openid-configuration - if (baseUri.equals(LOCALHOST_BASE_URI) || baseUri.equals(LOCALHOST_ORCID_WEB_BASE_URI) || m.matches()) { - if (crossDomainWebManger.allowed(request)) { - String origin = request.getHeader("origin"); - response.addHeader("Access-Control-Allow-Origin", origin); - response.addHeader("Access-Control-Allow-Credentials", "true"); - - if (request.getHeader("Access-Control-Request-Method") != null && "OPTIONS".equals(request.getMethod())) { - // CORS "pre-flight" request - response.addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE"); - response.addHeader("Access-Control-Allow-Headers", "X-Requested-With,Origin,Content-Type,Accept,Authorization,x-csrf-token"); - } + if (crossDomainWebManger.allowed(request)) { + String origin = request.getHeader("origin"); + response.addHeader("Access-Control-Allow-Origin", origin); + response.addHeader("Access-Control-Allow-Credentials", "true"); + + if (request.getHeader("Access-Control-Request-Method") != null && "OPTIONS".equals(request.getMethod())) { + // CORS "pre-flight" request + response.addHeader("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE"); + response.addHeader("Access-Control-Allow-Headers", "X-Requested-With,Origin,Content-Type,Accept,Authorization,x-csrf-token"); } - } filterChain.doFilter(request, response); diff --git a/properties/development.properties b/properties/development.properties index 9186b22ceb..f492b78131 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -99,7 +99,7 @@ org.orcid.core.node = 1 org.orcid.core.numberOfNodes = 1 # CORS allowed domains -org.orcid.security.cors.allowed_domains=localhost +org.orcid.security.cors.allowed_domains=dev.orcid.org # Messaging # Replace with tcp://domain.com:61616 in live to point at ActiveMQ location @@ -260,8 +260,8 @@ org.orcid.core.autospam.webhookUrl= org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr - org.orcid.persistence.panoply.cleanup.production=false + # Panoply redshift database org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver org.orcid.core.utils.panoply.maxPoolSize=20 From 32f3b3bfbdbc7c304f1a22a40ee32aa52aba1527 Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 30 Oct 2024 13:13:00 +0000 Subject: [PATCH 05/12] v2.67.2 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ca3420c95..cceb5e9446 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.2 - 2024-10-30 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.1...v2.67.2) + ## v2.67.1 - 2024-10-29 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.0...v2.67.1) From 739fc89798856e9ce0710dd78834bee884506fc9 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:51:13 +0000 Subject: [PATCH 06/12] Api limiting http request filter (#7110) * Implementation of API Rate Limiting Filter * Update development.properties * Bug fixes for slack reporting * Added the properties for test file * Added the tracking to panoply for anonymous/known users that exceeded the rate limit * changed props names * Update ApiRateLimitFilter.java to remove system.out * ApiRateLimitFilter.java log as trace when started * Update orcid-t1-web-context.xml --------- Co-authored-by: Angel Montenegro --- .../java/org/orcid/core/togglz/Features.java | 5 +- .../core/template/papi_rate_limit_email.ftl | 19 ++ .../template/papi_rate_limit_email_html.ftl | 36 +++ .../dao/PublicApiDailyRateLimitDao.java | 15 ++ .../impl/PublicApiDailyRateLimitDaoImpl.java | 126 +++++++++ .../PublicApiDailyRateLimitEntity.java | 129 ++++++++++ .../main/resources/META-INF/persistence.xml | 2 + .../src/main/resources/db-master.xml | 1 + .../create_public_api_daily_rate_limit.xml | 54 ++++ .../resources/orcid-persistence-context.xml | 4 + .../orcid/api/filters/ApiRateLimitFilter.java | 240 ++++++++++++++++++ .../resources/orcid-t1-security-context.xml | 7 +- .../main/resources/orcid-t1-web-context.xml | 14 +- orcid-pub-web/src/main/webapp/WEB-INF/web.xml | 10 + .../report/PapiDailyLimitReport.java | 100 ++++++++ .../resources/orcid-scheduler-context.xml | 3 + .../properties/test-scheduler.properties | 8 + .../PanoplyPapiDailyRateExceededItem.java | 49 ++++ .../utils/panoply/PanoplyRedshiftClient.java | 8 +- properties/development.properties | 13 +- 20 files changed, 832 insertions(+), 11 deletions(-) create mode 100644 orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl create mode 100644 orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl create mode 100644 orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java create mode 100644 orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java create mode 100644 orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java create mode 100644 orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml create mode 100644 orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java create mode 100644 orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java create mode 100644 orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java diff --git a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java index 389a28d2dc..d975b43664 100644 --- a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java +++ b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java @@ -53,7 +53,10 @@ public enum Features implements Feature { EMAIL_DOMAINS, @Label("Enable email domains in the UI") - EMAIL_DOMAINS_UI; + EMAIL_DOMAINS_UI, + + @Label("Enforce rate limiting for public API when disabled the rate monitoring is on. When disabled is the mode is monitoring only.") + ENABLE_PAPI_RATE_LIMITING; public boolean isActive() { return FeatureContext.getFeatureManager().isActive(this); diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl new file mode 100644 index 0000000000..0c4345237c --- /dev/null +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email.ftl @@ -0,0 +1,19 @@ +<#import "email_macros.ftl" as emailMacros /> +Dear ${emailName}, + +This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration: + +Client Name: ${clientName} +Client ID: ${clientId} + +Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service + +If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations. + +To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options. + +Warm Regards, +ORCID Support Team +https://support.orcid.org +<@emailMacros.msg "email.common.you_have_received_this_email" /> +<#include "email_footer.ftl"/> diff --git a/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl new file mode 100644 index 0000000000..569cb86ebe --- /dev/null +++ b/orcid-core/src/main/resources/org/orcid/core/template/papi_rate_limit_email_html.ftl @@ -0,0 +1,36 @@ +<#import "email_macros.ftl" as emailMacros /> +<#escape x as x?html> + + + + ${subject} + + +
+ ORCID.org +
+ Dear ${emailName}, +

This is an important message to let you know that you have exceeded our daily Public API usage limit with your integration:

+

Client Name: ${clientName}

+

Client ID: ${clientId}

+
+

Please remember that the ORCID Public API is free for non-commercial use by individuals as stated in the Public APIs Terms of Service (https://info.orcid.org/public-client-terms-of-service/). By “non-commercial” we mean that you may not charge any re-use fees for the Public API, and you may not make use of the Public API in connection with any revenue-generating product or service.

+

If you need access to an ORCID API for commercial use, need a higher usage quota, organizational administration of your API credentials, or the ability to write data to or access Trusted Party data in ORCID records, our Member API (https://info.orcid.org/documentation/features/member-api/) is available to ORCID member organizations.

+

To minimize any disruption to your ORCID integration in the future, we would recommend that you reach out to our Engagement Team by replying to this email to discuss our ORCID membership options. +
+

+ Warm Regards, + ORCID Support Team + https://support.orcid.org +

+ +

+ <@emailMacros.msg "email.common.you_have_received_this_email" /> +

+

+ <#include "email_footer_html.ftl"/> +

+
+ + + diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java new file mode 100644 index 0000000000..6d09069079 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/PublicApiDailyRateLimitDao.java @@ -0,0 +1,15 @@ +package org.orcid.persistence.dao; + +import java.time.LocalDate; + +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; + +public interface PublicApiDailyRateLimitDao extends GenericDao { + + PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate); + PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate); + int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit); + int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit); + boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient); + +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java new file mode 100644 index 0000000000..75f7154925 --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java @@ -0,0 +1,126 @@ +package org.orcid.persistence.dao.impl; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import javax.persistence.Query; + +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.transaction.annotation.Transactional; + +public class PublicApiDailyRateLimitDaoImpl extends GenericDaoImpl implements PublicApiDailyRateLimitDao { + private static final Logger LOG = LoggerFactory.getLogger(PublicApiDailyRateLimitDaoImpl.class); + + public PublicApiDailyRateLimitDaoImpl() { + super(PublicApiDailyRateLimitEntity.class); + } + + @Override + public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) { + Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate", + PublicApiDailyRateLimitEntity.class); + nativeQuery.setParameter("clientId", clientId); + nativeQuery.setParameter("requestDate", requestDate); + List papiRateList = (List) nativeQuery.getResultList(); + if (papiRateList != null && papiRateList.size() > 0) { + if (papiRateList.size() > 1) { + LOG.warn("Found more than one entry for the daily papi rate limiting the client: " + clientId + " and request date: " + requestDate.toString()); + } + return (PublicApiDailyRateLimitEntity) papiRateList.get(0); + } + return null; + } + + @Override + public PublicApiDailyRateLimitEntity findByIpAddressAndRequestDate(String ipAddress, LocalDate requestDate) { + String baseQuery = "SELECT * FROM public_api_daily_rate_limit p where p.ip_address=:ipAddress and p.request_date=:requestDate"; + + Query nativeQuery = entityManager.createNativeQuery(baseQuery, PublicApiDailyRateLimitEntity.class); + nativeQuery.setParameter("ipAddress", ipAddress); + nativeQuery.setParameter("requestDate", requestDate); + + List papiRateList = (List) nativeQuery.getResultList(); + if (papiRateList != null && papiRateList.size() > 0) { + LOG.debug("found results ...."); + if (papiRateList.size() > 1) { + LOG.warn("Found more than one entry for the daily papi rate limiting, the IP Address: " + ipAddress + " and request date: " + requestDate.toString()); + } + return (PublicApiDailyRateLimitEntity) papiRateList.get(0); + } + return null; + } + + public int countClientRequestsWithLimitExceeded(LocalDate requestDate, int limit) { + Query nativeQuery = entityManager.createNativeQuery( + "SELECT count(*) FROM public_api_daily_rate_limit p WHERE NOT ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount"); + nativeQuery.setParameter("requestDate", requestDate); + nativeQuery.setParameter("requestCount", limit); + List tsList = nativeQuery.getResultList(); + if (tsList != null && !tsList.isEmpty()) { + return tsList.get(0).intValue(); + } + return 0; + + } + + public int countAnonymousRequestsWithLimitExceeded(LocalDate requestDate, int limit) { + Query nativeQuery = entityManager.createNativeQuery( + "SELECT count(*) FROM public_api_daily_rate_limit p WHERE ((p.client_id = '' OR p.client_id IS NULL)) and p.request_date=:requestDate and p.request_count >=:requestCount"); + nativeQuery.setParameter("requestDate", requestDate); + nativeQuery.setParameter("requestCount", limit); + List tsList = nativeQuery.getResultList(); + if (tsList != null && !tsList.isEmpty()) { + return tsList.get(0).intValue(); + } + return 0; + } + + @Override + @Transactional + public boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiRateLimitingEntity, boolean isClient) { + Query query; + if (isClient) { + query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where " + + "client_id = :clientId and request_date =:requestDate"); + query.setParameter("clientId", papiRateLimitingEntity.getClientId()); + } else { + query = entityManager.createNativeQuery("update public_api_daily_rate_limit set request_count = :requestCount, last_modified = now() where " + + "ip_address = :ipAddress and request_date =:requestDate"); + query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress()); + } + query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount()); + query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate().toString()); + return query.executeUpdate() > 0; + } + + @Override + @Transactional + public void persist(PublicApiDailyRateLimitEntity papiRateLimitingEntity) { + String insertQuery = "INSERT INTO public_api_daily_rate_limit " + "(id, client_id, ip_address, request_count, request_date, date_created, last_modified)" + + " VALUES ( NEXTVAL('papi_daily_limit_seq'), :clientId , :ipAddress, :requestCount," + " :requestDate, now(), now())"; + + Query query = entityManager.createNativeQuery(insertQuery); + query.setParameter("clientId", papiRateLimitingEntity.getClientId()); + query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress()); + query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount()); + query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate()); + query.executeUpdate(); + return; + } + + private static String logQueryWithParams(String baseQuery, Map params) { + for (Map.Entry entry : params.entrySet()) { + String paramPlaceholder = ":" + entry.getKey(); + String paramValue = (entry.getValue() instanceof String) ? "'" + entry.getValue() + "'" : entry.getValue().toString(); + baseQuery = baseQuery.replace(paramPlaceholder, paramValue); + } + return baseQuery; + } + +} diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java new file mode 100644 index 0000000000..bc8a9d42ec --- /dev/null +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java @@ -0,0 +1,129 @@ +package org.orcid.persistence.jpa.entities; + +import java.io.Serializable; +import java.time.LocalDate; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.EntityManager; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.PrePersist; +import javax.persistence.PreUpdate; +import javax.persistence.SequenceGenerator; +import javax.persistence.Table; + + +@Entity +@Table(name = "public_api_daily_rate_limit") +public class PublicApiDailyRateLimitEntity implements OrcidEntity{ + + private static final long serialVersionUID = 7137838021634312424L; + + @Id + @GeneratedValue(strategy = GenerationType.AUTO, generator = "papi_daily_limit_seq") + @SequenceGenerator(name = "papi_daily_limit_seq", sequenceName = "papi_daily_limit_seq", allocationSize = 1) + private Long id; + + @Column(name = "client_id", nullable = true) + private String clientId; + + @Column(name = "ip_address", nullable = true) + private String ipAddress; + + @Column(name = "request_count", nullable = false) + private Long requestCount; + + @Column(name = "request_date", nullable = false) + private LocalDate requestDate; + + @Column(name = "date_created", nullable = false) + private Date dateCreated; + + @Column(name = "last_modified", nullable = false) + private Date lastModified; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getClientId() { + return clientId; + } + + public void setClientId(String clientId) { + this.clientId = clientId; + } + + public String getIpAddress() { + return ipAddress; + } + + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + + public Long getRequestCount() { + return requestCount; + } + + public void setRequestCount(Long requestCount) { + this.requestCount = requestCount; + } + + public LocalDate getRequestDate() { + return requestDate; + } + + public void setRequestDate(LocalDate requestDate) { + this.requestDate = requestDate; + } + + + public Date getDateCreated() { + return dateCreated; + } + + void setDateCreated(Date date) { + this.dateCreated = date; + } + + public Date getLastModified() { + return lastModified; + } + + void setLastModified(Date lastModified) { + this.lastModified = lastModified; + } + + @PreUpdate + void preUpdate() { + lastModified = new Date(); + } + + @PrePersist + void prePersist() { + Date now = new Date(); + dateCreated = now; + lastModified = now; + } + + public static > Map mapById(Collection entities) { + Map map = new HashMap(entities.size()); + for (E entity : entities) { + map.put(entity.getId(), entity); + } + return map; + } + +} + diff --git a/orcid-persistence/src/main/resources/META-INF/persistence.xml b/orcid-persistence/src/main/resources/META-INF/persistence.xml index e92cdc61d7..dda9237067 100644 --- a/orcid-persistence/src/main/resources/META-INF/persistence.xml +++ b/orcid-persistence/src/main/resources/META-INF/persistence.xml @@ -96,6 +96,8 @@ org.orcid.statistics.jpa.entities.StatisticValuesEntity org.orcid.statistics.jpa.entities.StatisticKeyEntity + + org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity true diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml index 65dfe060bf..a4a3b726fd 100644 --- a/orcid-persistence/src/main/resources/db-master.xml +++ b/orcid-persistence/src/main/resources/db-master.xml @@ -405,4 +405,5 @@ + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml b/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml new file mode 100644 index 0000000000..1b575f020d --- /dev/null +++ b/orcid-persistence/src/main/resources/db/updates/create_public_api_daily_rate_limit.xml @@ -0,0 +1,54 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SELECT 1 FROM pg_roles WHERE rolname='orcidro' + + GRANT SELECT ON public_api_daily_rate_limit TO orcidro; + + + \ No newline at end of file diff --git a/orcid-persistence/src/main/resources/orcid-persistence-context.xml b/orcid-persistence/src/main/resources/orcid-persistence-context.xml index 629b3b163c..03fce2ff5d 100644 --- a/orcid-persistence/src/main/resources/orcid-persistence-context.xml +++ b/orcid-persistence/src/main/resources/orcid-persistence-context.xml @@ -482,4 +482,8 @@ + + + + diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java new file mode 100644 index 0000000000..4a6105494d --- /dev/null +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -0,0 +1,240 @@ +package org.orcid.api.filters; + +import java.io.IOException; +import java.io.PrintWriter; +import java.time.LocalDate; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +import javax.annotation.Resource; +import javax.servlet.FilterChain; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.lang.LocaleUtils; +import org.orcid.core.manager.ClientDetailsEntityCacheManager; +import org.orcid.core.manager.OrcidSecurityManager; +import org.orcid.core.manager.TemplateManager; +import org.orcid.core.manager.impl.OrcidUrlManager; +import org.orcid.core.manager.v3.EmailManager; +import org.orcid.core.manager.v3.RecordNameManager; +import org.orcid.core.utils.OrcidRequestUtil; +import org.orcid.persistence.dao.ProfileDao; +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; +import org.orcid.persistence.jpa.entities.ClientDetailsEntity; +import org.orcid.persistence.jpa.entities.ProfileEntity; +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; +import org.orcid.utils.email.MailGunManager; +import org.orcid.utils.panoply.PanoplyPapiDailyRateExceededItem; +import org.orcid.utils.panoply.PanoplyRedshiftClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.stereotype.Component; +import org.springframework.web.filter.OncePerRequestFilter; + +import org.orcid.core.togglz.Features; + +@Component +public class ApiRateLimitFilter extends OncePerRequestFilter { + private static Logger LOG = LoggerFactory.getLogger(ApiRateLimitFilter.class); + + @Autowired + private PublicApiDailyRateLimitDao papiRateLimitingDao; + + @Autowired + private OrcidSecurityManager orcidSecurityManager; + + @Autowired + private ClientDetailsEntityCacheManager clientDetailsEntityCacheManager; + + @Autowired + private MailGunManager mailGunManager; + + @Autowired + private ProfileDao profileDao; + + @Autowired + private OrcidUrlManager orcidUrlManager; + + @Autowired + private RecordNameManager recordNameManager; + + @Autowired + private TemplateManager templateManager; + + @Autowired + private EmailManager emailManager; + + @Resource + private PanoplyRedshiftClient panoplyClient; + + @Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}") + private int anonymousRequestLimit; + + @Value("${org.orcid.papi.rate.limit.known.requests:40000}") + private int knownRequestLimit; + + @Value("${org.orcid.papi.rate.limit.enabled:false}") + private boolean enableRateLimiting; + + @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") + private boolean enablePanoplyPapiExceededRateInProduction; + + private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n" + + "You can increase your daily quota by registering for and using Public API client credentials " + + "(https://info.orcid.org/documentation/integration-guide/registering-a-public-api-client/ )"; + + private static final String SUBJECT = "[ORCID] You have exceeded the daily Public API Usage Limit - "; + private static final String FROM_ADDRESS = "\"Engagement Team, ORCID\" "; + + @Override + protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, FilterChain filterChain) + throws ServletException, IOException { + LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); + if (enableRateLimiting) { + String clientId = orcidSecurityManager.getClientIdFromAPIRequest(); + String ipAddress = OrcidRequestUtil.getIpAddress(httpServletRequest); + boolean isAnonymous = (clientId == null); + LocalDate today = LocalDate.now(); + + if (isAnonymous) { + LOG.info("ApiRateLimitFilter anonymous request"); + this.rateLimitAnonymousRequest(ipAddress, today, httpServletResponse); + + } else { + LOG.info("ApiRateLimitFilter client request with clientId: " + clientId); + this.rateLimitClientRequest(clientId, today); + } + + filterChain.doFilter(httpServletRequest, httpServletResponse); + } + } + + private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException { + PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByIpAddressAndRequestDate(ipAddress, today); + if (rateLimitEntity != null) { + // update the request count only when limit not exceeded ? + rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, false); + if (rateLimitEntity.getRequestCount() == knownRequestLimit && enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setIpAddress(ipAddress); + item.setRequestDate(rateLimitEntity.getRequestDate()); + setPapiRateExceededItemInPanoply(item); + } + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { + if (rateLimitEntity.getRequestCount() >= anonymousRequestLimit) { + httpServletResponse.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + if (!httpServletResponse.isCommitted()) { + try (PrintWriter writer = httpServletResponse.getWriter()) { + writer.write(TOO_MANY_REQUESTS_MSG); + writer.flush(); + } + return; + } + } + } + } else { + // create + rateLimitEntity = new PublicApiDailyRateLimitEntity(); + rateLimitEntity.setIpAddress(ipAddress); + rateLimitEntity.setRequestCount(1L); + rateLimitEntity.setRequestDate(today); + papiRateLimitingDao.persist(rateLimitEntity); + + } + return; + + } + + private void rateLimitClientRequest(String clientId, LocalDate today) { + PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByClientIdAndRequestDate(clientId, today); + if (rateLimitEntity != null) { + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { + // email the client first time the limit is reached + if (rateLimitEntity.getRequestCount() == knownRequestLimit) { + sendEmail(clientId, rateLimitEntity.getRequestDate()); + } + } + // update the request count + rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); + papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity,true); + + } else { + // create + rateLimitEntity = new PublicApiDailyRateLimitEntity(); + rateLimitEntity.setClientId(clientId); + rateLimitEntity.setRequestCount(0L); + rateLimitEntity.setRequestDate(today); + papiRateLimitingDao.persist(rateLimitEntity); + } + + + } + + private Map createTemplateParams(String clientId, String clientName, String emailName, String orcidId) { + Map templateParams = new HashMap(); + templateParams.put("clientId", clientId); + templateParams.put("clientId", clientName); + templateParams.put("emailName", emailName); + templateParams.put("locale", LocaleUtils.toLocale("en")); + templateParams.put("baseUri", orcidUrlManager.getBaseUrl()); + templateParams.put("baseUriHttp", orcidUrlManager.getBaseUriHttp()); + templateParams.put("subject", SUBJECT + orcidId); + return templateParams; + } + + private void sendEmail(String clientId, LocalDate requestDate) { + ClientDetailsEntity clientDetailsEntity = clientDetailsEntityCacheManager.retrieve(clientId); + ProfileEntity profile = profileDao.find(clientDetailsEntity.getGroupProfileId()); + String emailName = recordNameManager.deriveEmailFriendlyName(profile.getId()); + Map templateParams = this.createTemplateParams(clientId, clientDetailsEntity.getClientName(), emailName, profile.getId()); + // Generate body from template + String body = templateManager.processTemplate("bad_orgs_email.ftl", templateParams); + // Generate html from template + String html = templateManager.processTemplate("bad_orgs_email_html.ftl", templateParams); + String email = emailManager.findPrimaryEmail(profile.getId()).getEmail(); + + LOG.info("text email={}", body); + LOG.info("html email={}", html); + if (enablePanoplyPapiExceededRateInProduction) { + PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); + item.setClientId(clientId); + item.setOrcid(profile.getId()); + item.setEmail(email); + item.setRequestDate(requestDate); + setPapiRateExceededItemInPanoply(item); + } + + // Send the email + boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html); + if (!mailSent) { + throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId()); + } + } + + + private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) { + //Store the rate exceeded item in panoply Db without blocking + CompletableFuture.supplyAsync(() -> { + try { + panoplyClient.addPanoplyPapiDailyRateExceeded(item); + return true; + } catch (Exception e) { + LOG.error("Cannot store the rateExceededItem to panoply ", e); + return false; + } + }).thenAccept(result -> { + if(! result) { + LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result); + } + + }); + } + +} diff --git a/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml b/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml index 87ebab08ca..73fe82eef5 100644 --- a/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml +++ b/orcid-pub-web/src/main/resources/orcid-t1-security-context.xml @@ -82,12 +82,13 @@ - - - + + \ No newline at end of file diff --git a/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml b/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml index 60a2db946f..4189a06e33 100644 --- a/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml +++ b/orcid-pub-web/src/main/resources/orcid-t1-web-context.xml @@ -16,16 +16,18 @@ - + - + - - - + + + + + @@ -86,4 +88,6 @@ + + diff --git a/orcid-pub-web/src/main/webapp/WEB-INF/web.xml b/orcid-pub-web/src/main/webapp/WEB-INF/web.xml index 1688d337a9..aded7e0765 100644 --- a/orcid-pub-web/src/main/webapp/WEB-INF/web.xml +++ b/orcid-pub-web/src/main/webapp/WEB-INF/web.xml @@ -135,6 +135,16 @@ clientCredentialsTokenEndpointFilter /oauth/token + + + rateLimitingFilter + org.springframework.web.filter.DelegatingFilterProxy + + + + rateLimitingFilter + * + springSecurityFilterChain diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java new file mode 100644 index 0000000000..0e63e4f905 --- /dev/null +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -0,0 +1,100 @@ +package org.orcid.scheduler.report; + +import java.time.LocalDate; + +import javax.annotation.Resource; + +import org.orcid.core.togglz.Features; +import org.orcid.core.togglz.OrcidTogglzConfiguration; +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; + +import org.orcid.utils.alerting.SlackManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.ApplicationContext; +import org.springframework.context.support.ClassPathXmlApplicationContext; +import org.springframework.stereotype.Service; +import org.togglz.core.context.ContextClassLoaderFeatureManagerProvider; +import org.togglz.core.manager.FeatureManager; +import org.togglz.core.manager.FeatureManagerBuilder; + +@Service +public class PapiDailyLimitReport { + + private static final Logger LOG = LoggerFactory.getLogger(PapiDailyLimitReport.class); + + @Resource + private SlackManager slackManager; + + @Value("${org.orcid.core.papiLimitReport.slackChannel:collab-spam-reports}") + private String slackChannel; + + @Value("${org.orcid.core.papiLimitReport.webhookUrl}") + private String webhookUrl; + + @Value("${org.orcid.core.orgs.load.slackUser}") + private String slackUser; + + @Value("${rate.limit.anonymous.requests}") + private int anonymousRequestLimit; + + @Value("${rate.limit.known.requests}") + private int knownRequestLimit; + + @Value("${rate.limit.enabled:false}") + private boolean enableRateLimiting; + + @Autowired + private PublicApiDailyRateLimitDao papiRateLimitingDao; + + // for running spam manually + public static void main(String[] args) { + PapiDailyLimitReport dailyLimitReport = new PapiDailyLimitReport(); + try { + dailyLimitReport.init(); + dailyLimitReport.papiDailyLimitReport(); + } catch (Exception e) { + LOG.error("Exception when getting the report for daily limit", e); + System.err.println(e.getMessage()); + } finally { + System.exit(0); + } + + } + /** + * Sends daily slack reports to dedicated slack channel. + */ + public void papiDailyLimitReport() { + LOG .info("start papi limit report the rate limiting is: " + enableRateLimiting); + if (enableRateLimiting) { + LocalDate yesterday = LocalDate.now().minusDays(1); + String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; + String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\n Current Anonymous Requests Limit: " + anonymousRequestLimit + + "\n Current Public API Clients Limit: " + knownRequestLimit + "\n Mode: " + mode; + LOG .info(SLACK_INTRO_MSG); + slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl); + + String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit) + + "\n Count of Public API clients that have exceeded the limit: " + + papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit); + LOG .info(SLACK_STATS_MSG); + slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl); + } + + } + + @SuppressWarnings("resource") + private void init() { + ApplicationContext context = new ClassPathXmlApplicationContext("orcid-scheduler-context.xml"); + papiRateLimitingDao = (PublicApiDailyRateLimitDao) context.getBean("papiRateLimitingDao"); + bootstrapTogglz(context.getBean(OrcidTogglzConfiguration.class)); + } + + private static void bootstrapTogglz(OrcidTogglzConfiguration togglzConfig) { + FeatureManager featureManager = new FeatureManagerBuilder().togglzConfig(togglzConfig).build(); + ContextClassLoaderFeatureManagerProvider.bind(featureManager); + } + +} diff --git a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml index 897d1e1a90..dc6dd87b42 100644 --- a/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml +++ b/orcid-scheduler-web/src/main/resources/orcid-scheduler-context.xml @@ -45,6 +45,7 @@ + @@ -138,5 +139,7 @@ + + diff --git a/orcid-test/src/main/resources/properties/test-scheduler.properties b/orcid-test/src/main/resources/properties/test-scheduler.properties index f1d0663669..796e8862ba 100644 --- a/orcid-test/src/main/resources/properties/test-scheduler.properties +++ b/orcid-test/src/main/resources/properties/test-scheduler.properties @@ -54,3 +54,11 @@ org.orcid.core.utils.panoply.idleConnectionTimeout=3600 org.orcid.core.utils.panoply.connectionTimeout=36000 org.orcid.core.utils.panoply.jdbcUrl=xxx org.orcid.core.utils.panoply.username=xxx + +#Slack channel for papi limits +org.orcid.core.papiLimitReport.slackChannel=xxx +org.orcid.core.papiLimitReport.webhookUrl=xxx +rate.limit.anonymous.requests=1 +rate.limit.known.requests=2 +rate.limit.enabled=true +org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * \ No newline at end of file diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java new file mode 100644 index 0000000000..9761ed95f4 --- /dev/null +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyPapiDailyRateExceededItem.java @@ -0,0 +1,49 @@ +package org.orcid.utils.panoply; + +import java.time.LocalDate; + +public class PanoplyPapiDailyRateExceededItem { + private String ipAddress; + private String clientId; + private String email; + private String orcid; + private LocalDate requestDate; + + + public String getIpAddress() { + return ipAddress; + } + public void setIpAddress(String ipAddress) { + this.ipAddress = ipAddress; + } + public String getClientId() { + return clientId; + } + public void setClientId(String clientId) { + this.clientId = clientId; + } + public String getEmail() { + return email; + } + public void setEmail(String email) { + this.email = email; + } + public String getOrcid() { + return orcid; + } + public void setOrcid(String orcid) { + this.orcid = orcid; + } + public LocalDate getRequestDate() { + return requestDate; + } + public void setRequestDate(LocalDate requestDate) { + this.requestDate = requestDate; + } + + @Override + public String toString() { + return "PanoplyPapiDailyRateExceededItem{" + "ipAddress=" + ipAddress + ", clientId='" + clientId + '\'' + ", email='" + email + '\'' + ", orcid='" + orcid + '\'' + + ", requestDate='" + requestDate + '\'' + '}'; + } +} diff --git a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java index 77c5db8ef1..a7b0d24002 100644 --- a/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java +++ b/orcid-utils/src/main/java/org/orcid/utils/panoply/PanoplyRedshiftClient.java @@ -31,5 +31,11 @@ public int addPanoplyDeletedItem(PanoplyDeletedItem item) { return panoplyJdbcTemplate.update(sql, item.getItemId(), item.getOrcid(), item.getClientSourceId(), new java.sql.Timestamp(new Date().getTime()), item.getDwTable()); } - + + public int addPanoplyPapiDailyRateExceeded(PanoplyPapiDailyRateExceededItem item) { + LOG.debug("Adding papi daily rate exceeded item to panoply DB: " + item.toString()); + String sql = "INSERT INTO dw_papi_daily_rate_exceeded (ip_address, orcid, client_id, email, request_date) VALUES (?, ?, ?, ?, ?)"; + return panoplyJdbcTemplate.update(sql, item.getIpAddress(), item.getOrcid(), item.getClientId(), item.getEmail(), item.getRequestDate()); + } + } diff --git a/properties/development.properties b/properties/development.properties index f492b78131..036dd07a4a 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -258,9 +258,11 @@ org.orcid.scheduler.autospam.enabled=true org.orcid.core.autospam.slackChannel=collab-spam-reports org.orcid.core.autospam.webhookUrl= -org.orcid.persistence.liquibase.enabled=false +#org.orcid.persistence.liquibase.enabled=false org.orcid.persistence.solr.read.only.url=http://localhost:8983/solr org.orcid.persistence.panoply.cleanup.production=false +#api_limiting_http_request_filter +org.orcid.persistence.panoply.papiExceededRate.production=false # Panoply redshift database org.orcid.core.utils.panoply.driver=com.amazon.redshift.jdbc.Driver @@ -268,5 +270,14 @@ org.orcid.core.utils.panoply.maxPoolSize=20 org.orcid.core.utils.panoply.password=xxx org.orcid.core.utils.panoply.idleConnectionTimeout=3600 org.orcid.core.utils.panoply.connectionTimeout=36000 +#jdbc:redshift://:/ org.orcid.core.utils.panoply.jdbcUrl=xxx org.orcid.core.utils.panoply.username=xxx + +#Slack channel for papi limits +org.orcid.core.papiLimitReport.slackChannel=system-alerts-qa +org.orcid.core.papiLimitReport.webhookUrl=xxx +org.orcid.papi.rate.limit.anonymous.requests=1 +org.orcid.papi.rate.limit.known.requests=2 +org.orcid.papi.rate.limit.enabled=true +org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * \ No newline at end of file From 3528f7f046940aec2d4daa6bd8f0619263d1936c Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 30 Oct 2024 14:07:03 +0000 Subject: [PATCH 07/12] v2.67.3 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cceb5e9446..aa6eacc9d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.3 - 2024-10-30 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.2...v2.67.3) + ## v2.67.2 - 2024-10-30 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.1...v2.67.2) From ee4f96c8cde64c60a3a882289db98266dbb8deac Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Wed, 30 Oct 2024 14:09:08 +0000 Subject: [PATCH 08/12] Added the missing v2 attributes when creating orgs for ROR type (#7116) Co-authored-by: Angel Montenegro --- .../orcid/scheduler/loader/source/ror/RorOrgLoadSource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java index 28da2f9a72..d815dc672c 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/loader/source/ror/RorOrgLoadSource.java @@ -281,7 +281,7 @@ private OrgDisambiguatedEntity processInstitute(String sourceId, String name, Is } // Create a new disambiguated org - OrgDisambiguatedEntity newOrg = createDisambiguatedOrg(sourceId, name, orgType, country, city, region, url); + OrgDisambiguatedEntity newOrg = createDisambiguatedOrg(sourceId, name, orgType, country, city, region, url, locationsJson, namesJson); try { // mark group for indexing new OrgGrouping(newOrg, orgDisambiguatedManager).markGroupForIndexing(orgDisambiguatedDao); @@ -394,7 +394,7 @@ private boolean entityChanged(OrgDisambiguatedEntity entity, String name, String /** * Creates a disambiguated ORG in the org_disambiguated table */ - private OrgDisambiguatedEntity createDisambiguatedOrg(String sourceId, String name, String orgType, Iso3166Country country, String city, String region, String url) { + private OrgDisambiguatedEntity createDisambiguatedOrg(String sourceId, String name, String orgType, Iso3166Country country, String city, String region, String url,String locationsJson, String namesJson) { LOGGER.info("Creating disambiguated org {}", name); OrgDisambiguatedEntity orgDisambiguatedEntity = new OrgDisambiguatedEntity(); orgDisambiguatedEntity.setName(name); @@ -405,6 +405,8 @@ private OrgDisambiguatedEntity createDisambiguatedOrg(String sourceId, String na orgDisambiguatedEntity.setOrgType(orgType); orgDisambiguatedEntity.setSourceId(sourceId); orgDisambiguatedEntity.setSourceType(OrgDisambiguatedSourceType.ROR.name()); + orgDisambiguatedEntity.setLocationsJson(locationsJson); + orgDisambiguatedEntity.setNamesJson(namesJson); orgDisambiguatedManager.createOrgDisambiguated(orgDisambiguatedEntity); return orgDisambiguatedEntity; } From f3aa7c605309c9276051ea759f16be4e2d811ce4 Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 30 Oct 2024 14:24:32 +0000 Subject: [PATCH 09/12] v2.67.4 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa6eacc9d1..bedbb02408 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.4 - 2024-10-30 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.3...v2.67.4) + ## v2.67.3 - 2024-10-30 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.2...v2.67.3) From c413162eb458ccaf2d327583decbc8ff341735fa Mon Sep 17 00:00:00 2001 From: Camelia Dumitru <62257307+Camelia-Orcid@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:13:51 +0000 Subject: [PATCH 10/12] Fixing the SQL Error in QA (#7117) --- .../persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java index 75f7154925..589811693b 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java @@ -95,7 +95,7 @@ public boolean updatePublicApiDailyRateLimit(PublicApiDailyRateLimitEntity papiR query.setParameter("ipAddress", papiRateLimitingEntity.getIpAddress()); } query.setParameter("requestCount", papiRateLimitingEntity.getRequestCount()); - query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate().toString()); + query.setParameter("requestDate", papiRateLimitingEntity.getRequestDate()); return query.executeUpdate() > 0; } From 6a3979078509271134876e4982f57db26648172f Mon Sep 17 00:00:00 2001 From: github actions Date: Wed, 30 Oct 2024 16:27:32 +0000 Subject: [PATCH 11/12] v2.67.5 changelog update --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bedbb02408..f1df19b0c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## v2.67.5 - 2024-10-30 + +[Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.4...v2.67.5) + ## v2.67.4 - 2024-10-30 [Full Changelog](https://github.com/ORCID/ORCID-Source/compare/v2.67.3...v2.67.4) From d9eb4969912c497bc64ffaa112ba73d036e7775f Mon Sep 17 00:00:00 2001 From: andrej romanov <50377758+auumgn@users.noreply.github.com> Date: Thu, 31 Oct 2024 17:05:49 +0200 Subject: [PATCH 12/12] add source name and or id to emails missing both (#7118) * add source name and or id to emails missing both * add unfinished test * Remove dup bean definition from child object * add tests * fix tests * update email count in test --------- Co-authored-by: amontenegro --- .../controllers/ManageProfileController.java | 15 +- .../controllers/PublicRecordController.java | 12 +- .../ManageProfileControllerTest.java | 134 ++++++++++++++++-- 3 files changed, 140 insertions(+), 21 deletions(-) diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java index e3e99fb1b3..72997bfb7b 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/ManageProfileController.java @@ -94,9 +94,6 @@ public class ManageProfileController extends BaseWorkspaceController { @Resource private GivenPermissionToManager givenPermissionToManager; - @Resource(name = "emailManagerReadOnlyV3") - private EmailManagerReadOnly emailManagerReadOnly; - @Resource(name = "profileEmailDomainManagerReadOnly") private ProfileEmailDomainManagerReadOnly profileEmailDomainManagerReadOnly; @@ -536,7 +533,17 @@ public ModelAndView confirmDeactivateOrcidAccount(HttpServletRequest request, Ht if (Features.EMAIL_DOMAINS.isActive()) { emailDomains = profileEmailDomainManagerReadOnly.getEmailDomains(getCurrentUserOrcid()); } - return org.orcid.pojo.ajaxForm.Emails.valueOf(v2Emails, emailDomains); + org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(v2Emails, emailDomains); + // Old emails are missing the source name and id -- assign the user as the source + for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + String orcid = getCurrentUserOrcid(); + String displayName = getPersonDetails(orcid, true).getDisplayName(); + email.setSource(orcid); + email.setSourceName(displayName); + } + } + return emails; } @RequestMapping(value = "/emails.json", method = RequestMethod.POST) diff --git a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java index feb8126519..b078751344 100644 --- a/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java +++ b/orcid-web/src/main/java/org/orcid/frontend/web/controllers/PublicRecordController.java @@ -202,8 +202,16 @@ PublicRecord getRecord(String orcid) { if (Features.EMAIL_DOMAINS.isActive()) { emailDomains = profileEmailDomainManagerReadOnly.getPublicEmailDomains(orcid); } - - publicRecord.setEmails(org.orcid.pojo.ajaxForm.Emails.valueOf(filteredEmails, emailDomains)); + + org.orcid.pojo.ajaxForm.Emails emails = org.orcid.pojo.ajaxForm.Emails.valueOf(filteredEmails, emailDomains); + // Old emails are missing the source name and id -- assign the user as the source + for (org.orcid.pojo.ajaxForm.Email email: emails.getEmails()) { + if (email.getSource() == null && email.getSourceName() == null) { + email.setSource(orcid); + email.setSourceName(publicRecord.getDisplayName()); + } + } + publicRecord.setEmails(emails); // Fill external identifiers PersonExternalIdentifiers publicPersonExternalIdentifiers; diff --git a/orcid-web/src/test/java/org/orcid/frontend/web/controllers/ManageProfileControllerTest.java b/orcid-web/src/test/java/org/orcid/frontend/web/controllers/ManageProfileControllerTest.java index a89a4fa8d3..d501f59d48 100644 --- a/orcid-web/src/test/java/org/orcid/frontend/web/controllers/ManageProfileControllerTest.java +++ b/orcid-web/src/test/java/org/orcid/frontend/web/controllers/ManageProfileControllerTest.java @@ -40,17 +40,13 @@ import org.orcid.core.manager.v3.OrcidSecurityManager; import org.orcid.core.manager.v3.ProfileEntityManager; import org.orcid.core.manager.v3.RecordNameManager; -import org.orcid.core.manager.v3.read_only.GivenPermissionToManagerReadOnly; -import org.orcid.core.manager.v3.read_only.ProfileEntityManagerReadOnly; -import org.orcid.core.manager.v3.read_only.RecordNameManagerReadOnly; +import org.orcid.core.manager.v3.read_only.*; import org.orcid.core.oauth.OrcidProfileUserDetails; import org.orcid.core.security.OrcidWebRole; +import org.orcid.jaxb.model.v3.release.common.*; import org.orcid.utils.DateUtils; import org.orcid.core.utils.v3.OrcidIdentifierUtils; import org.orcid.frontend.email.RecordEmailSender; -import org.orcid.jaxb.model.v3.release.common.CreditName; -import org.orcid.jaxb.model.v3.release.common.OrcidIdentifier; -import org.orcid.jaxb.model.v3.release.common.Visibility; import org.orcid.jaxb.model.v3.release.record.Biography; import org.orcid.jaxb.model.v3.release.record.Email; import org.orcid.jaxb.model.v3.release.record.Emails; @@ -84,6 +80,7 @@ public class ManageProfileControllerTest { private static final String USER_ORCID = "0000-0000-0000-0001"; private static final String DEPRECATED_USER_ORCID = "0000-0000-0000-0002"; private static final String DEPRECATED_USER_ORCID_URL = "https://localhost:8443/0000-0000-0000-0002"; + private static final String USER_CREDIT_NAME = "Credit Name"; @Mock private ProfileEntityCacheManager mockProfileEntityCacheManager; @@ -94,6 +91,9 @@ public class ManageProfileControllerTest { @Mock private EmailManager mockEmailManager; + @Mock + private ProfileEmailDomainManagerReadOnly mockProfileEmailDomainManagerReadOnly; + @Mock private LocaleManager mockLocaleManager; @@ -127,6 +127,21 @@ public class ManageProfileControllerTest { @Mock(name="profileEntityManagerReadOnlyV3") private ProfileEntityManagerReadOnly mockProfileEntityManagerReadOnly; + @Mock + private PersonalDetailsManagerReadOnly mockPersonalDetailsManagerReadOnly; + + @Mock + private AddressManagerReadOnly mockAddressManagerReadOnly; + + @Mock + private ProfileKeywordManagerReadOnly mockKeywordManagerReadOnly; + + @Mock + private ResearcherUrlManagerReadOnly mockResearcherUrlManagerReadOnly; + + @Mock + private ExternalIdentifierManagerReadOnly mockExternalIdentifierManagerReadOnly; + @Before public void initMocks() throws Exception { controller = new ManageProfileController(); @@ -136,6 +151,7 @@ public void initMocks() throws Exception { TargetProxyHelper.injectIntoProxy(controller, "encryptionManager", mockEncryptionManager); TargetProxyHelper.injectIntoProxy(controller, "emailManager", mockEmailManager); TargetProxyHelper.injectIntoProxy(controller, "emailManagerReadOnly", mockEmailManager); + TargetProxyHelper.injectIntoProxy(controller, "profileEmailDomainManagerReadOnly", mockProfileEmailDomainManagerReadOnly); TargetProxyHelper.injectIntoProxy(controller, "localeManager", mockLocaleManager); TargetProxyHelper.injectIntoProxy(controller, "profileEntityManager", mockProfileEntityManager); TargetProxyHelper.injectIntoProxy(controller, "givenPermissionToManager", mockGivenPermissionToManager); @@ -147,6 +163,12 @@ public void initMocks() throws Exception { TargetProxyHelper.injectIntoProxy(controller, "twoFactorAuthenticationManager", twoFactorAuthenticationManager); TargetProxyHelper.injectIntoProxy(controller, "recordEmailSender", mockRecordEmailSender); TargetProxyHelper.injectIntoProxy(controller, "profileEntityManagerReadOnly", mockProfileEntityManagerReadOnly); + TargetProxyHelper.injectIntoProxy(controller, "personalDetailsManagerReadOnly", mockPersonalDetailsManagerReadOnly); + TargetProxyHelper.injectIntoProxy(controller, "addressManagerReadOnly", mockAddressManagerReadOnly); + TargetProxyHelper.injectIntoProxy(controller, "keywordManagerReadOnly", mockKeywordManagerReadOnly); + TargetProxyHelper.injectIntoProxy(controller, "researcherUrlManagerReadOnly", mockResearcherUrlManagerReadOnly); + TargetProxyHelper.injectIntoProxy(controller, "externalIdentifierManagerReadOnly", mockExternalIdentifierManagerReadOnly); + when(mockOrcidSecurityManager.isPasswordConfirmationRequired()).thenReturn(true); when(mockEncryptionManager.hashMatches(Mockito.anyString(), Mockito.anyString())).thenReturn(true); @@ -192,13 +214,23 @@ public Emails answer(InvocationOnMock invocation) throws Throwable { Emails emails = new Emails(); Email email1 = new Email(); email1.setEmail(invocation.getArgument(0) + "_1@test.orcid.org"); + email1.setSource(new Source()); email1.setVisibility(Visibility.PUBLIC); emails.getEmails().add(email1); Email email2 = new Email(); email2.setEmail(invocation.getArgument(0) + "_2@test.orcid.org"); + email2.setSource(new Source()); + email2.getSource().setSourceName(new SourceName(USER_CREDIT_NAME)); email2.setVisibility(Visibility.PUBLIC); emails.getEmails().add(email2); + + Email email3 = new Email(); + email3.setEmail(invocation.getArgument(0) + "_3@test.orcid.org"); + email3.setSource(new Source()); + email3.getSource().setSourceClientId(new SourceClientId(USER_ORCID)); + email3.setVisibility(Visibility.PUBLIC); + emails.getEmails().add(email3); return emails; } @@ -387,16 +419,18 @@ public void testValidateDeprecateProfileWithValidData() { assertNotNull(deprecateProfile.getDeprecatingEmails()); assertEquals("0000-0000-0000-0002", deprecateProfile.getDeprecatingOrcid()); assertEquals("0000-0000-0000-0002 Given Names 0000-0000-0000-0002 Family Name", deprecateProfile.getDeprecatingAccountName()); - assertEquals(2, deprecateProfile.getDeprecatingEmails().size()); + assertEquals(3, deprecateProfile.getDeprecatingEmails().size()); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_1@test.orcid.org")); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_2@test.orcid.org")); + assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_3@test.orcid.org")); assertEquals("0000-0000-0000-0001", deprecateProfile.getPrimaryOrcid()); assertEquals("0000-0000-0000-0001 Given Names 0000-0000-0000-0001 Family Name", deprecateProfile.getPrimaryAccountName()); assertNotNull(deprecateProfile.getPrimaryEmails()); - assertEquals(2, deprecateProfile.getPrimaryEmails().size()); + assertEquals(3, deprecateProfile.getPrimaryEmails().size()); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_1@test.orcid.org")); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_2@test.orcid.org")); + assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_3@test.orcid.org")); assertTrue(deprecateProfile.getErrors().isEmpty()); // Using orcid @@ -409,16 +443,19 @@ public void testValidateDeprecateProfileWithValidData() { assertNotNull(deprecateProfile.getDeprecatingEmails()); assertEquals("0000-0000-0000-0002", deprecateProfile.getDeprecatingOrcid()); assertEquals("0000-0000-0000-0002 Given Names 0000-0000-0000-0002 Family Name", deprecateProfile.getDeprecatingAccountName()); - assertEquals(2, deprecateProfile.getDeprecatingEmails().size()); + assertEquals(3, deprecateProfile.getDeprecatingEmails().size()); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_1@test.orcid.org")); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_2@test.orcid.org")); + assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_3@test.orcid.org")); + assertEquals("0000-0000-0000-0001", deprecateProfile.getPrimaryOrcid()); assertEquals("0000-0000-0000-0001 Given Names 0000-0000-0000-0001 Family Name", deprecateProfile.getPrimaryAccountName()); assertNotNull(deprecateProfile.getPrimaryEmails()); - assertEquals(2, deprecateProfile.getPrimaryEmails().size()); + assertEquals(3, deprecateProfile.getPrimaryEmails().size()); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_1@test.orcid.org")); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_2@test.orcid.org")); + assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_3@test.orcid.org")); assertTrue(deprecateProfile.getErrors().isEmpty()); // Using orcid URL @@ -431,16 +468,19 @@ public void testValidateDeprecateProfileWithValidData() { assertNotNull(deprecateProfile.getDeprecatingEmails()); assertEquals("0000-0000-0000-0002", deprecateProfile.getDeprecatingOrcid()); assertEquals("0000-0000-0000-0002 Given Names 0000-0000-0000-0002 Family Name", deprecateProfile.getDeprecatingAccountName()); - assertEquals(2, deprecateProfile.getDeprecatingEmails().size()); + assertEquals(3, deprecateProfile.getDeprecatingEmails().size()); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_1@test.orcid.org")); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_2@test.orcid.org")); + assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_3@test.orcid.org")); assertEquals("0000-0000-0000-0001", deprecateProfile.getPrimaryOrcid()); assertEquals("0000-0000-0000-0001 Given Names 0000-0000-0000-0001 Family Name", deprecateProfile.getPrimaryAccountName()); assertNotNull(deprecateProfile.getPrimaryEmails()); - assertEquals(2, deprecateProfile.getPrimaryEmails().size()); + assertEquals(3, deprecateProfile.getPrimaryEmails().size()); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_1@test.orcid.org")); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_2@test.orcid.org")); + assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_3@test.orcid.org")); + assertTrue(deprecateProfile.getErrors().isEmpty()); // Using orcid trim space @@ -454,16 +494,19 @@ public void testValidateDeprecateProfileWithValidData() { assertEquals("0000-0000-0000-0002", deprecateProfile.getDeprecatingOrcid()); assertEquals("0000-0000-0000-0002 Given Names 0000-0000-0000-0002 Family Name", deprecateProfile.getDeprecatingAccountName()); - assertEquals(2, deprecateProfile.getDeprecatingEmails().size()); + assertEquals(3, deprecateProfile.getDeprecatingEmails().size()); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_1@test.orcid.org")); assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_2@test.orcid.org")); + assertTrue(deprecateProfile.getDeprecatingEmails().contains("0000-0000-0000-0002_3@test.orcid.org")); + assertEquals("0000-0000-0000-0001", deprecateProfile.getPrimaryOrcid()); assertEquals("0000-0000-0000-0001 Given Names 0000-0000-0000-0001 Family Name", deprecateProfile.getPrimaryAccountName()); assertNotNull(deprecateProfile.getPrimaryEmails()); - assertEquals(2, deprecateProfile.getPrimaryEmails().size()); + assertEquals(3, deprecateProfile.getPrimaryEmails().size()); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_1@test.orcid.org")); assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_2@test.orcid.org")); + assertTrue(deprecateProfile.getPrimaryEmails().contains("0000-0000-0000-0001_3@test.orcid.org")); assertTrue(deprecateProfile.getErrors().isEmpty()); } @@ -1116,7 +1159,68 @@ public void testEditEmail_primaryEmailChange() { verify(mockRecordEmailSender, Mockito.times(1)).sendVerificationEmail(eq(USER_ORCID), eq("email@orcid.org"), eq(true)); } - + + @Test + public void testEmptyEmailSource() { + SecurityContextHolder.getContext().setAuthentication(getAuthentication(USER_ORCID)); + when(mockProfileEmailDomainManagerReadOnly.getEmailDomains(eq(USER_ORCID))).thenReturn(null); + when(mockEmailManager.getPublicEmails(eq(USER_ORCID))).thenReturn(new Emails()); + MockHttpServletRequest mockRequest = new MockHttpServletRequest(); + MockHttpSession mockSession = new MockHttpSession(); + mockRequest.setSession(mockSession); + org.orcid.pojo.ajaxForm.Emails emails = controller.getEmails(mockRequest); + + assertEquals(3, emails.getEmails().size()); + + org.orcid.pojo.ajaxForm.Email email1 = emails.getEmails().get(0); + assertEquals(email1.getValue(), USER_ORCID + "_1@test.orcid.org"); + assertEquals(email1.getSource(), USER_ORCID); + assertNull(email1.getSourceName()); + + org.orcid.pojo.ajaxForm.Email email2 = emails.getEmails().get(1); + assertEquals(email2.getValue(), USER_ORCID + "_2@test.orcid.org"); + assertNull(email2.getSource()); + assertEquals(email2.getSourceName(), USER_CREDIT_NAME); + } + + @Test + public void testEmailSourceWithSourceName() { + SecurityContextHolder.getContext().setAuthentication(getAuthentication(USER_ORCID)); + when(mockProfileEmailDomainManagerReadOnly.getEmailDomains(eq(USER_ORCID))).thenReturn(null); + when(mockEmailManager.getPublicEmails(eq(USER_ORCID))).thenReturn(new Emails()); + MockHttpServletRequest mockRequest = new MockHttpServletRequest(); + MockHttpSession mockSession = new MockHttpSession(); + mockRequest.setSession(mockSession); + org.orcid.pojo.ajaxForm.Emails emails = controller.getEmails(mockRequest); + + assertEquals(3, emails.getEmails().size()); + + org.orcid.pojo.ajaxForm.Email email2 = emails.getEmails().get(1); + assertEquals(email2.getValue(), USER_ORCID + "_2@test.orcid.org"); + assertNull(email2.getSource()); + assertEquals(email2.getSourceName(), USER_CREDIT_NAME); + } + + @Test + public void testEmailSourceWithSourceId() { + SecurityContextHolder.getContext().setAuthentication(getAuthentication(USER_ORCID)); + when(mockProfileEmailDomainManagerReadOnly.getEmailDomains(eq(USER_ORCID))).thenReturn(null); + when(mockEmailManager.getPublicEmails(eq(USER_ORCID))).thenReturn(new Emails()); + MockHttpServletRequest mockRequest = new MockHttpServletRequest(); + MockHttpSession mockSession = new MockHttpSession(); + mockRequest.setSession(mockSession); + org.orcid.pojo.ajaxForm.Emails emails = controller.getEmails(mockRequest); + + assertEquals(3, emails.getEmails().size()); + + org.orcid.pojo.ajaxForm.Email email3 = emails.getEmails().get(2); + assertEquals(email3.getValue(), USER_ORCID + "_3@test.orcid.org"); + assertNull(email3.getSourceName()); + assertEquals(email3.getSource(), USER_ORCID); + } + + + protected Authentication getAuthentication(String orcid) { List roles = Arrays.asList(OrcidWebRole.ROLE_USER); OrcidProfileUserDetails details = new OrcidProfileUserDetails(orcid, "user_1@test.orcid.org", null, roles);