Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fixes for papi limit filter and ML start #7121

Merged
merged 3 commits into from
Nov 4, 2024
Merged

Fixes for papi limit filter and ML start #7121

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ea0d73
Fixes for papi limit filter and ML start
Camelia-Orcid Nov 4, 2024
62f11c1
Added the exception, fixed formatting
Camelia-Orcid Nov 4, 2024
ab03e71
Merge branch 'main' into papi_limit_filter_qa_errors
amontenegro Nov 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStore.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,4 +25,6 @@ public interface OrcidTokenStore extends TokenStore {
OAuth2Authentication readAuthenticationFromCachedToken(Map<String, String> cachedTokenData);

void isClientEnabled(String clientId) throws InvalidTokenException;

String readClientId(String tokenValue);
}
10 changes: 10 additions & 0 deletions orcid-core/src/main/java/org/orcid/core/oauth/service/OrcidTokenStoreServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,16 @@ public OrcidOauth2TokenDetail readOrcidOauth2TokenDetail(String token) {
return orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token);
}

@Override
public String readClientId(String token) {
String clientId = null;
OrcidOauth2TokenDetail orcidTokenDetail = orcidOauthTokenDetailService.findIgnoringDisabledByTokenValue(token);
if(orcidTokenDetail != null) {
clientId = orcidTokenDetail.getClientDetailsId();
}
return clientId;
}

/**
* Read the authentication stored under the specified token value.
*
Expand Down
14 changes: 7 additions & 7 deletions orcid-message-listener/src/main/resources/orcid-message-listener-web-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,13 +161,13 @@

<!-- Panoply Redshift -->
<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver:com.amazon.redshift.jdbc.Driver}" />
<property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="\${org.orcid.core.utils.panoply.username}" />
<property name="password" value="\${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize:5}" />
<property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout:3600}" />
<property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout:3600}" />
<property name="driverClassName" value="${org.orcid.core.utils.panoply.driver:com.amazon.redshift.jdbc.Driver}" />
<property name="jdbcUrl" value="${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="${org.orcid.core.utils.panoply.username}" />
<property name="password" value="${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="${org.orcid.core.utils.panoply.maxPoolSize:5}" />
<property name="idleTimeout" value="${org.orcid.core.utils.panoply.idleConnectionTimeout:3600}" />
<property name="connectionTimeout" value="${org.orcid.core.utils.panoply.connectionTimeout:3600}" />
</bean>

<!-- Panoply JdbcTemplate Bean Definition -->
Expand Down
2 changes: 1 addition & 1 deletion ...sistence/src/main/java/org/orcid/persistence/dao/impl/PublicApiDailyRateLimitDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public PublicApiDailyRateLimitDaoImpl() {

@Override
public PublicApiDailyRateLimitEntity findByClientIdAndRequestDate(String clientId, LocalDate requestDate) {
Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p client_id=:clientId and requestDate=:requestDate",
Query nativeQuery = entityManager.createNativeQuery("SELECT * FROM public_api_daily_rate_limit p where p.client_id=:clientId and p.request_date=:requestDate",
PublicApiDailyRateLimitEntity.class);
nativeQuery.setParameter("clientId", clientId);
nativeQuery.setParameter("requestDate", requestDate);
Expand Down
14 changes: 7 additions & 7 deletions orcid-persistence/src/main/resources/orcid-persistence-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,13 +460,13 @@

<!-- Panoply Redshift -->
<bean id="panoplyRedshiftDataSource" class="com.zaxxer.hikari.HikariDataSource" destroy-method="close">
<property name="driverClassName" value="\${org.orcid.core.utils.panoply.driver:com.amazon.redshift.jdbc.Driver}" />
<property name="jdbcUrl" value="\${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="\${org.orcid.core.utils.panoply.username}" />
<property name="password" value="\${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="\${org.orcid.core.utils.panoply.maxPoolSize:5}" />
<property name="idleTimeout" value="\${org.orcid.core.utils.panoply.idleConnectionTimeout:3600}" />
<property name="connectionTimeout" value="\${org.orcid.core.utils.panoply.connectionTimeout:3600}" />
<property name="driverClassName" value="${org.orcid.core.utils.panoply.driver:com.amazon.redshift.jdbc.Driver}" />
<property name="jdbcUrl" value="${org.orcid.core.utils.panoply.jdbcUrl}" />
<property name="username" value="${org.orcid.core.utils.panoply.username}" />
<property name="password" value="${org.orcid.core.utils.panoply.password}" />
<property name="maximumPoolSize" value="${org.orcid.core.utils.panoply.maxPoolSize:5}" />
<property name="idleTimeout" value="${org.orcid.core.utils.panoply.idleConnectionTimeout:3600}" />
<property name="connectionTimeout" value="${org.orcid.core.utils.panoply.connectionTimeout:3600}" />
</bean>

<!-- Panoply JdbcTemplate Bean Definition -->
Expand Down
42 changes: 26 additions & 16 deletions orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.orcid.core.manager.impl.OrcidUrlManager;
import org.orcid.core.manager.v3.EmailManager;
import org.orcid.core.manager.v3.RecordNameManager;
import org.orcid.core.oauth.service.OrcidTokenStore;
import org.orcid.core.utils.OrcidRequestUtil;
import org.orcid.persistence.dao.ProfileDao;
import org.orcid.persistence.dao.PublicApiDailyRateLimitDao;
Expand Down Expand Up @@ -69,9 +70,12 @@ public class ApiRateLimitFilter extends OncePerRequestFilter {

@Autowired
private EmailManager emailManager;

@Resource
private PanoplyRedshiftClient panoplyClient;
private PanoplyRedshiftClient panoplyClient;

@Autowired
private OrcidTokenStore orcidTokenStore;

@Value("${org.orcid.papi.rate.limit.anonymous.requests:10000}")
private int anonymousRequestLimit;
Expand All @@ -81,9 +85,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter {

@Value("${org.orcid.papi.rate.limit.enabled:false}")
private boolean enableRateLimiting;

@Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}")
private boolean enablePanoplyPapiExceededRateInProduction;
private boolean enablePanoplyPapiExceededRateInProduction;

private static final String TOO_MANY_REQUESTS_MSG = "Too Many Requests - You have exceeded the daily allowance of API calls.\\n"
+ "You can increase your daily quota by registering for and using Public API client credentials "
Expand All @@ -97,8 +101,16 @@ protected void doFilterInternal(HttpServletRequest httpServletRequest, HttpServl
throws ServletException, IOException {
LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting);
if (enableRateLimiting) {
String clientId = orcidSecurityManager.getClientIdFromAPIRequest();
String ipAddress = OrcidRequestUtil.getIpAddress(httpServletRequest);
String tokenValue = httpServletRequest.getHeader("Authorization").replaceAll("Bearer|bearer", "").trim();

String ipAddress = httpServletRequest.getRemoteAddr();

String clientId = null;
try {
clientId = orcidTokenStore.readClientId(tokenValue);
} catch (Exception ex) {
LOG.error("Exception when trying to get the client id from token value, ignoring and treating as anonymous client", ex);
}
boolean isAnonymous = (clientId == null);
LocalDate today = LocalDate.now();

Expand Down Expand Up @@ -163,17 +175,16 @@ private void rateLimitClientRequest(String clientId, LocalDate today) {
}
// update the request count
rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1);
papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity,true);
papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, true);

} else {
// create
rateLimitEntity = new PublicApiDailyRateLimitEntity();
rateLimitEntity.setClientId(clientId);
rateLimitEntity.setRequestCount(0L);
rateLimitEntity.setRequestCount(1L);
rateLimitEntity.setRequestDate(today);
papiRateLimitingDao.persist(rateLimitEntity);
}


}

Expand Down Expand Up @@ -212,15 +223,14 @@ private void sendEmail(String clientId, LocalDate requestDate) {
}

// Send the email
boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email , SUBJECT, body, html);
boolean mailSent = mailGunManager.sendEmail(FROM_ADDRESS, email, SUBJECT, body, html);
if (!mailSent) {
throw new RuntimeException("Failed to send email for papi limits, orcid=" + profile.getId());
}
}



private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem item) {
//Store the rate exceeded item in panoply Db without blocking
// Store the rate exceeded item in panoply Db without blocking
CompletableFuture.supplyAsync(() -> {
try {
panoplyClient.addPanoplyPapiDailyRateExceeded(item);
Expand All @@ -229,9 +239,9 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i
LOG.error("Cannot store the rateExceededItem to panoply ", e);
return false;
}
}).thenAccept(result -> {
if(! result) {
LOG.error("Async call to panoply for : " + item.toString() + " Stored: "+ result);
}).thenAccept(result -> {
if (!result) {
LOG.error("Async call to panoply for : " + item.toString() + " Stored: " + result);
}

});
Expand Down
Loading