Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feature: Start collection daily pub api stats #7016

Merged
merged 4 commits into from
Apr 4, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

import javax.servlet.http.HttpServletRequest;

import org.orcid.core.utils.EventType;
import org.orcid.persistence.jpa.entities.EventType;

/**
*
Expand All @@ -13,4 +13,6 @@ public interface EventManager {

void createEvent(EventType eventType, HttpServletRequest request);

void createPapiEvent(String clientId, boolean anonymous);

}
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,12 @@
import org.orcid.core.constants.OrcidOauth2Constants;
import org.orcid.core.manager.ClientDetailsEntityCacheManager;
import org.orcid.core.manager.v3.read_only.RecordNameManagerReadOnly;
import org.orcid.core.utils.EventType;
import org.orcid.jaxb.model.clientgroup.ClientType;
import org.orcid.jaxb.model.v3.release.record.Name;
import org.orcid.persistence.dao.EventDao;
import org.orcid.persistence.jpa.entities.ClientDetailsEntity;
import org.orcid.persistence.jpa.entities.EventEntity;
import org.orcid.persistence.jpa.entities.EventType;
import org.orcid.pojo.ajaxForm.PojoUtil;
import org.orcid.pojo.ajaxForm.RequestInfoForm;

Expand Down Expand Up @@ -105,4 +105,14 @@ private String removeAttributesFromUrl(String url) {
}
return url;
}

@Override
public void createPapiEvent(String clientId, boolean anonymous) {
EventEntity eventEntity = new EventEntity();
eventEntity.setEventType(EventType.PAPI.getValue());
eventEntity.setClientId(clientId);
eventEntity.setLabel(anonymous ? "anonymous" : null);
eventEntity.setDateCreated(new Date());
eventDao.createEvent(eventEntity);
}
}
5 changes: 4 additions & 1 deletion orcid-core/src/main/java/org/orcid/core/togglz/Features.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,10 @@ public enum Features implements Feature {
SEND_ADD_WORKS_EMAILS,

@Label("Delete events older than 90 days from the DB ")
DELETE_EVENTS;
DELETE_EVENTS,

@Label("Track public events stats ")
PAPI_EVENTS;

public boolean isActive() {
return FeatureContext.getFeatureManager().isActive(this);
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.orcid.persistence.dao;

import org.orcid.persistence.jpa.entities.EventEntity;
import org.orcid.persistence.jpa.entities.EventType;

import java.util.List;

Expand All @@ -18,6 +19,10 @@ public interface EventDao {
void delete(long id);

List<EventEntity> findAll();

List<EventEntity> findByEventType(EventType eventType);

void deleteEventsByDate(Integer numberOfDays);

void deletePapiEvents(Integer numberOfDays);
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package org.orcid.persistence.dao;

import org.orcid.persistence.jpa.entities.EventStatsEntity;
import org.orcid.persistence.jpa.entities.EventType;

import java.util.List;

Expand All @@ -13,5 +14,9 @@ public interface EventStatsDao {

void createEventStats();

void createPapiEventStats();

List<EventStatsEntity> findAll();

void deleteStatsByType(EventType eventType);
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

import org.orcid.persistence.dao.EventDao;
import org.orcid.persistence.jpa.entities.EventEntity;
import org.orcid.persistence.jpa.entities.EventType;
import org.springframework.transaction.annotation.Transactional;

import java.util.List;
Expand Down Expand Up @@ -49,9 +50,27 @@ public List<EventEntity> findAll() {
@Override
@Transactional
public void deleteEventsByDate(Integer numberOfDays) {
String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date)";
String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type != :eventType";
Query queryDelete = entityManager.createNativeQuery(query);
queryDelete.setParameter("eventType", EventType.PAPI.getValue());
queryDelete.setParameter("numberOfDays", numberOfDays);
queryDelete.executeUpdate();
}

@Override
@Transactional
public void deletePapiEvents(Integer numberOfDays) {
String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type = :eventType";
Query queryDelete = entityManager.createNativeQuery(query);
queryDelete.setParameter("eventType", EventType.PAPI.getValue());
queryDelete.setParameter("numberOfDays", numberOfDays);
queryDelete.executeUpdate();
}

@Override
public List<EventEntity> findByEventType(EventType eventType) {
TypedQuery<EventEntity> query = entityManager.createQuery("from EventEntity where eventType = :eventType", EventEntity.class);
query.setParameter("eventType", eventType.getValue());
return query.getResultList();
}
}
Original file line number Diff line number Diff line change
@@ -1,13 +1,17 @@
package org.orcid.persistence.dao.impl;

import org.orcid.persistence.dao.EventStatsDao;
import org.orcid.persistence.jpa.entities.EventType;
import org.orcid.persistence.jpa.entities.EventStatsEntity;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Resource;
import javax.persistence.EntityManager;
import javax.persistence.Query;
import javax.persistence.TypedQuery;

import java.math.BigInteger;
import java.util.ArrayList;
import java.util.List;

/**
Expand All @@ -25,9 +29,11 @@ public void createEventStats() {
"INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) " +
"SELECT event_type, client_id, COUNT(id), CAST(e.date_created as date), now(), now() " +
"FROM event as e " +
"WHERE CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
"WHERE event_type != '"+ EventType.PAPI.getValue() + "' " +
"AND CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
"GROUP BY event_type, client_id, CAST(e.date_created as date) " +
"ORDER BY CAST(e.date_created as date) DESC;";

Query insertQuery = entityManager.createNativeQuery(query);
insertQuery.executeUpdate();
}
Expand All @@ -37,4 +43,48 @@ public List<EventStatsEntity> findAll() {
TypedQuery<EventStatsEntity> query = entityManager.createQuery("from EventStatsEntity", EventStatsEntity.class);
return query.getResultList();
}

@Override
@Transactional
public void createPapiEventStats() {
String query =
"SELECT event_type, client_id, label, count(*), CAST(date_created as date), now(), now() " +
"FROM event " +
"WHERE event_type = '"+ EventType.PAPI.getValue() + "' " +
"AND CAST(date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
"GROUP BY event_type, client_id, label, CAST(date_created as date) " +
"ORDER BY CAST(date_created as date) DESC;";

Query queryList = entityManager.createNativeQuery(query);
List<Object[]> eventsList = queryList.getResultList();
List<Object[]> eventsListToRemove = new ArrayList<>();
if (eventsList.size() > 0) {
eventsList.forEach(item -> {
if (item[2] == "anonymous" && item[3] != null && ((BigInteger) item[3]).intValue() < 1000) {
eventsListToRemove.add(item);
}
});
eventsList.removeAll(eventsListToRemove);
eventsList.forEach(item -> {
String insertQuery = "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) VALUES (:eventType, :clientId, :count, :date, :dateCreated, :lastModified)";
Query insertQueryClients = entityManager.createNativeQuery(insertQuery);
insertQueryClients.setParameter("eventType", item[0]);
insertQueryClients.setParameter("clientId", item[1]);
insertQueryClients.setParameter("count", item[3]);
insertQueryClients.setParameter("date", item[4]);
insertQueryClients.setParameter("dateCreated", item[5]);
insertQueryClients.setParameter("lastModified", item[6]);
insertQueryClients.executeUpdate();
});
}
}

@Override
@Transactional
public void deleteStatsByType(EventType eventType) {
String query = "DELETE FROM event_stats where event_type = :eventType";
Query queryDelete = entityManager.createNativeQuery(query);
queryDelete.setParameter("eventType", eventType.getValue());
queryDelete.executeUpdate();
}
}
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package org.orcid.core.utils;
package org.orcid.persistence.jpa.entities;

public enum EventType {
SIGN_IN("Sign-In"),
NEW_REGISTRATION("New-Registration"),
AUTHORIZE("Authorize"),
AUTHORIZE_DENY("Authorize-Deny"),
REAUTHORIZE("Reauthorize"),
PUBLIC_PAGE("Public-Page");
PUBLIC_PAGE("Public-Page"),
PAPI("Public-API");

private final String value;

Expand All @@ -17,4 +18,4 @@ public enum EventType {
public String getValue() {
return value;
}
}
}
1 change: 1 addition & 0 deletions orcid-persistence/src/main/resources/db-master.xml
Original file line number Diff line number Diff line change
Expand Up @@ -385,4 +385,5 @@
<include file="/db/updates/dw_event_stats.xml" />
<include file="/db/updates/add_indexes_2024_02_20.xml" />
<include file="/db/updates/dw_views_with_4_months_interval.xml" />
<include file="/db/updates/dw_papi_event_stats.xml" />
</databaseChangeLog>
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">

<changeSet id="DROP-VIEW-FOR-EVENT" author="Daniel Palafox" dbms="postgresql">
<dropView viewName="dw_event_stats"/>
</changeSet>

<changeSet id="CREATE-DW-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
<preConditions onFail="MARK_RAN">
<not><viewExists viewName="dw_event_stats"/></not>
</preConditions>
<createView viewName="dw_event_stats">
SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
FROM event_stats WHERE event_type != 'Public-API'
ORDER BY DATE_TRUNC('day', date_created) DESC;
</createView>
</changeSet>

<changeSet id="GRANT-READ-TO-DW_USER-TO-DW_EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
<preConditions>
<sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
</preConditions>
<sql>GRANT SELECT ON TABLE dw_event_stats to dw_user;</sql>
</changeSet>

<changeSet id="CREATE-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
<preConditions onFail="MARK_RAN">
<not><viewExists viewName="dw_papi_event_stats"/></not>
</preConditions>
<createView viewName="dw_papi_event_stats">
SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
FROM event_stats WHERE event_type = 'Public-API'
ORDER BY DATE_TRUNC('day', date_created) DESC;
</createView>
</changeSet>

<changeSet id="GRANT-READ-TO-DW-USER-TO-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
<preConditions>
<sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
</preConditions>
<sql>GRANT SELECT ON TABLE dw_papi_event_stats to dw_user;</sql>
</changeSet>

</databaseChangeLog>
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.junit.Test;
import org.junit.runner.RunWith;
import org.orcid.persistence.jpa.entities.EventEntity;
import org.orcid.persistence.jpa.entities.EventType;
import org.orcid.test.DBUnitTest;
import org.orcid.test.OrcidJUnit4ClassRunner;
import org.springframework.test.context.ContextConfiguration;
Expand Down Expand Up @@ -81,4 +82,19 @@ public void testWriteEvent() throws IllegalAccessException {

eventDao.delete(eventEntity.getId());
}

@Test
public void deletePapiEventsByDate() {
List<EventEntity> eventEntityList = eventDao.findByEventType(EventType.PAPI);

assertNotNull(eventEntityList);
assertEquals(3, eventEntityList.size());

eventDao.deletePapiEvents(90);

eventEntityList = eventDao.findByEventType(EventType.PAPI);

assertNotNull(eventEntityList);
assertEquals(0, eventEntityList.size());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import org.junit.runner.RunWith;
import org.orcid.persistence.jpa.entities.EventEntity;
import org.orcid.persistence.jpa.entities.EventStatsEntity;
import org.orcid.persistence.jpa.entities.EventType;
import org.orcid.test.OrcidJUnit4ClassRunner;
import org.springframework.test.context.ContextConfiguration;

Expand Down Expand Up @@ -36,15 +37,69 @@ public void createEventStats() {
List<EventStatsEntity> eventStatsEntityList = eventStatsDao.findAll();

assertNotNull(eventStatsEntityList);
assertEquals(1, eventStatsEntityList.size());
assertEquals(2, eventStatsEntityList.size());
assertEquals(Integer.valueOf(20), eventStatsEntityList.get(0).getCount());
assertEquals(Integer.valueOf(20), eventStatsEntityList.get(1).getCount());

eventStatsDao.deleteStatsByType(EventType.SIGN_IN);
}

private void createEvents() {
@Test
public void createPapiEventStats() {
createPapiEvents();

eventStatsDao.createPapiEventStats();

List<EventStatsEntity> eventStatsEntityList = eventStatsDao.findAll();

assertNotNull(eventStatsEntityList);
assertEquals(3, eventStatsEntityList.size());
assertEquals(Integer.valueOf(10), eventStatsEntityList.get(0).getCount());
assertEquals(Integer.valueOf(10), eventStatsEntityList.get(1).getCount());
assertEquals(Integer.valueOf(1100), eventStatsEntityList.get(2).getCount());

eventStatsDao.deleteStatsByType(EventType.PAPI);
}

private void createPapiEvents() {
for (int i = 0; i < 20; i++) {
EventEntity eventEntity = new EventEntity();
eventEntity.setEventType("Sign-In");
eventEntity.setClientId("Client " + 1);
eventEntity.setEventType(EventType.PAPI.getValue());
eventEntity.setClientId("Client " + (i % 2 == 0 ? 1 : 2));
LocalDate date = LocalDate.now().minusDays(1);
Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
eventEntity.setDateCreated(Date.from(instant));
eventDao.createEvent(eventEntity);
}

for (int i = 0; i < 10; i++) {
EventEntity eventEntity = new EventEntity();
eventEntity.setEventType(EventType.PAPI.getValue());
eventEntity.setClientId("105.21.229.71");
eventEntity.setLabel("anonymous");
LocalDate date = LocalDate.now().minusDays(1);
Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
eventEntity.setDateCreated(Date.from(instant));
eventDao.createEvent(eventEntity);
}

for (int i = 0; i < 1100; i++) {
EventEntity eventEntity = new EventEntity();
eventEntity.setEventType(EventType.PAPI.getValue());
eventEntity.setClientId("104.20.228.70");
eventEntity.setLabel("anonymous");
LocalDate date = LocalDate.now().minusDays(1);
Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
eventEntity.setDateCreated(Date.from(instant));
eventDao.createEvent(eventEntity);
}
}

private void createEvents() {
for (int i = 0; i < 40; i++) {
EventEntity eventEntity = new EventEntity();
eventEntity.setEventType(EventType.SIGN_IN.getValue());
eventEntity.setClientId("Client " + (i % 2 == 0 ? 1 : 2)) ;
LocalDate date = LocalDate.now().minusDays(1);
Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
eventEntity.setDateCreated(Date.from(instant));
Expand Down
Loading
Loading