feature: Start collection daily pub api stats

orcid-core/src/main/java/org/orcid/core/common/manager/EventManager.java

@@ -2,7 +2,7 @@
 
 import javax.servlet.http.HttpServletRequest;
 
-import org.orcid.core.utils.EventType;
+import org.orcid.persistence.jpa.entities.EventType;
 
 /**
  *
@@ -13,4 +13,6 @@ public interface EventManager {
 
     void createEvent(EventType eventType, HttpServletRequest request);
 
+    void createPapiEvent(String clientId, boolean anonymous);
+
 }

orcid-core/src/main/java/org/orcid/core/common/manager/impl/EventManagerImpl.java

@@ -13,12 +13,12 @@
 import org.orcid.core.constants.OrcidOauth2Constants;
 import org.orcid.core.manager.ClientDetailsEntityCacheManager;
 import org.orcid.core.manager.v3.read_only.RecordNameManagerReadOnly;
-import org.orcid.core.utils.EventType;
 import org.orcid.jaxb.model.clientgroup.ClientType;
 import org.orcid.jaxb.model.v3.release.record.Name;
 import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.ClientDetailsEntity;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.pojo.ajaxForm.PojoUtil;
 import org.orcid.pojo.ajaxForm.RequestInfoForm;
 
@@ -105,4 +105,14 @@ private String removeAttributesFromUrl(String url) {
         }
         return url;
     }
+
+    @Override
+    public void createPapiEvent(String clientId, boolean anonymous) {
+        EventEntity eventEntity = new EventEntity();
+        eventEntity.setEventType(EventType.PAPI.getValue());
+        eventEntity.setClientId(clientId);
+        eventEntity.setLabel(anonymous ? "anonymous" : null);
+        eventEntity.setDateCreated(new Date());
+        eventDao.createEvent(eventEntity);
+    }
 }

orcid-core/src/main/java/org/orcid/core/togglz/Features.java

@@ -53,7 +53,10 @@ public enum Features implements Feature {
     SEND_ADD_WORKS_EMAILS,
 
     @Label("Delete events older than 90 days from the DB ")
-    DELETE_EVENTS;
+    DELETE_EVENTS,
+
+    @Label("Track public events stats ")
+    PAPI_EVENTS;
 
     public boolean isActive() {
         return FeatureContext.getFeatureManager().isActive(this);

orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java

@@ -1,6 +1,7 @@
 package org.orcid.persistence.dao;
 
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 
 import java.util.List;
 
@@ -18,6 +19,10 @@ public interface EventDao {
     void delete(long id);
 
     List<EventEntity> findAll();
+
+    List<EventEntity> findByEventType(EventType eventType);
 
     void deleteEventsByDate(Integer numberOfDays);
+
+    void deletePapiEvents(Integer numberOfDays);
 }

orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java

@@ -13,5 +13,7 @@ public interface EventStatsDao {
 
     void createEventStats();
 
+    void createPapiEventStats();
+
     List<EventStatsEntity> findAll();
 }

orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java

@@ -7,6 +7,7 @@
 
 import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
@@ -49,9 +50,27 @@ public List<EventEntity> findAll() {
     @Override
     @Transactional
     public void deleteEventsByDate(Integer numberOfDays) {
-        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date)";
+        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type != :eventType";
         Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("eventType", EventType.PAPI.getValue());
         queryDelete.setParameter("numberOfDays", numberOfDays);
         queryDelete.executeUpdate();
     }
+
+    @Override
+    @Transactional
+    public void deletePapiEvents(Integer numberOfDays) {
+        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type = :eventType";
+        Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("eventType", EventType.PAPI.getValue());
+        queryDelete.setParameter("numberOfDays", numberOfDays);
+        queryDelete.executeUpdate();
+    }
+
+    @Override
+    public List<EventEntity> findByEventType(EventType eventType) {
+        TypedQuery<EventEntity> query = entityManager.createQuery("from EventEntity where eventType = :eventType", EventEntity.class);
+        query.setParameter("eventType", eventType.getValue());
+        return query.getResultList();
+    }
 }

orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java

@@ -1,13 +1,17 @@
 package org.orcid.persistence.dao.impl;
 
 import org.orcid.persistence.dao.EventStatsDao;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.persistence.jpa.entities.EventStatsEntity;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -25,9 +29,11 @@ public void createEventStats() {
                 "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) " +
                 "SELECT event_type, client_id, COUNT(id), CAST(e.date_created as date), now(), now() " +
                 "FROM event as e " +
-                "WHERE CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
+                "WHERE event_type != '"+ EventType.PAPI.getValue() + "' " +
+                "AND CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
                 "GROUP BY event_type, client_id, CAST(e.date_created as date) " +
                 "ORDER BY CAST(e.date_created as date) DESC;";
+
         Query insertQuery = entityManager.createNativeQuery(query);
         insertQuery.executeUpdate();
     }
@@ -37,4 +43,39 @@ public List<EventStatsEntity> findAll() {
         TypedQuery<EventStatsEntity> query = entityManager.createQuery("from EventStatsEntity", EventStatsEntity.class);
         return query.getResultList();
     }
+
+    @Override
+    @Transactional
+    public void createPapiEventStats() {
+        String query = 
+                "SELECT event_type, client_id, label, count(*), CAST(date_created as date), now(), now() " +
+                "FROM event " +
+                "WHERE event_type = '"+ EventType.PAPI.getValue() + "' " +
+                "AND CAST(date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
+                "GROUP BY event_type, client_id, label, CAST(date_created as date) " +
+                "ORDER BY CAST(date_created as date) DESC;";
+
+        Query queryList = entityManager.createNativeQuery(query);
+        List<Object[]> eventsList = queryList.getResultList();
+        List<Object[]> eventsListToRemove = new ArrayList<>();
+        if (eventsList.size() > 0) {
+            eventsList.forEach(item -> {
+                if (item[2] == "anonymous" && item[3] != null && ((BigInteger) item[3]).intValue() < 1000) {
+                    eventsListToRemove.add(item);
+                }
+            });
+            eventsList.removeAll(eventsListToRemove);
+            eventsList.forEach(item -> {
+                String insertQuery = "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) VALUES (:eventType, :clientId, :count, :date, :dateCreated, :lastModified)";
+                Query insertQueryClients = entityManager.createNativeQuery(insertQuery);
+                insertQueryClients.setParameter("eventType", item[0]);
+                insertQueryClients.setParameter("clientId", item[1]);
+                insertQueryClients.setParameter("count", item[3]);
+                insertQueryClients.setParameter("date", item[4]);
+                insertQueryClients.setParameter("dateCreated", item[5]);
+                insertQueryClients.setParameter("lastModified", item[6]);
+                insertQueryClients.executeUpdate();
+            });
+        }
+    }
 }

orcid-core/src/main/java/org/orcid/core/utils/EventType.java → orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventType.java

@@ -1,12 +1,13 @@
-package org.orcid.core.utils;
+package org.orcid.persistence.jpa.entities;
 
 public enum EventType {
     SIGN_IN("Sign-In"),
     NEW_REGISTRATION("New-Registration"),
     AUTHORIZE("Authorize"),
     AUTHORIZE_DENY("Authorize-Deny"),
     REAUTHORIZE("Reauthorize"),
-    PUBLIC_PAGE("Public-Page");
+    PUBLIC_PAGE("Public-Page"),
+    PAPI("Public-API");
 
     private final String value;
 
@@ -17,4 +18,4 @@ public enum EventType {
     public String getValue() {
         return value;
     }
-}
+}

orcid-persistence/src/main/resources/db-master.xml

@@ -385,4 +385,5 @@
   <include file="/db/updates/dw_event_stats.xml" />  
   <include file="/db/updates/add_indexes_2024_02_20.xml" />
   <include file="/db/updates/dw_views_with_4_months_interval.xml" />
+  <include file="/db/updates/dw_papi_event_stats.xml" />  
 </databaseChangeLog>

orcid-persistence/src/main/resources/db/updates/dw_papi_event_stats.xml

@@ -0,0 +1,45 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+    <changeSet id="DROP-VIEW-FOR-EVENT" author="Daniel Palafox" dbms="postgresql">
+        <dropView viewName="dw_event_stats"/>
+    </changeSet>
+
+    <changeSet id="CREATE-DW-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions onFail="MARK_RAN">
+            <not><viewExists viewName="dw_event_stats"/></not>
+        </preConditions>
+        <createView viewName="dw_event_stats">
+            SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
+            FROM event_stats WHERE event_type != 'Public-API'
+            ORDER BY DATE_TRUNC('day', date_created) DESC;
+        </createView>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-TO-DW_USER-TO-DW_EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON TABLE dw_event_stats to dw_user;</sql>
+    </changeSet>
+
+    <changeSet id="CREATE-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions onFail="MARK_RAN">
+            <not><viewExists viewName="dw_papi_event_stats"/></not>
+        </preConditions>
+        <createView viewName="dw_papi_event_stats">
+            SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
+            FROM event_stats WHERE event_type = 'Public-API'
+            ORDER BY DATE_TRUNC('day', date_created) DESC;
+        </createView>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-TO-DW-USER-TO-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON TABLE dw_papi_event_stats to dw_user;</sql>
+    </changeSet>
+
+</databaseChangeLog>

orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java

@@ -17,6 +17,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.test.DBUnitTest;
 import org.orcid.test.OrcidJUnit4ClassRunner;
 import org.springframework.test.context.ContextConfiguration;
@@ -81,4 +82,19 @@ public void testWriteEvent() throws IllegalAccessException {
 
         eventDao.delete(eventEntity.getId());
     }
+
+    @Test
+    public void deletePapiEventsByDate() {
+        List<EventEntity> eventEntityList = eventDao.findByEventType(EventType.PAPI);
+
+        assertNotNull(eventEntityList);
+        assertEquals(3, eventEntityList.size());
+
+        eventDao.deletePapiEvents(90);
+
+        eventEntityList = eventDao.findByEventType(EventType.PAPI);
+
+        assertNotNull(eventEntityList);
+        assertEquals(0, eventEntityList.size());
+    }
 }

orcid-persistence/src/test/java/org/orcid/persistence/dao/EventStatsDaoTest.java

@@ -36,14 +36,64 @@ public void createEventStats() {
         List<EventStatsEntity> eventStatsEntityList = eventStatsDao.findAll();
 
         assertNotNull(eventStatsEntityList);
-        assertEquals(1, eventStatsEntityList.size());
+        assertEquals(2, eventStatsEntityList.size());
         assertEquals(Integer.valueOf(20), eventStatsEntityList.get(0).getCount());
+        assertEquals(Integer.valueOf(20), eventStatsEntityList.get(1).getCount());
     }
 
-    private void createEvents() {
+    @Test
+    public void createPapiEventStats() {
+        createPapiEvents();
+
+        eventStatsDao.createPapiEventStats();
+
+        List<EventStatsEntity> eventStatsEntityList = eventStatsDao.findAll();
+
+        assertNotNull(eventStatsEntityList);
+        assertEquals(3, eventStatsEntityList.size());
+        assertEquals(Integer.valueOf(10), eventStatsEntityList.get(0).getCount());
+        assertEquals(Integer.valueOf(10), eventStatsEntityList.get(1).getCount());
+        assertEquals(Integer.valueOf(1100), eventStatsEntityList.get(2).getCount());
+    }
+
+    private void createPapiEvents() {
         for (int i = 0; i < 20; i++) {
             EventEntity eventEntity = new EventEntity();
-            eventEntity.setEventType("Sign-In");
+            eventEntity.setEventType("Public-API");
+            eventEntity.setClientId("Client " + (i % 2 == 0 ? 1 : 2));
+            LocalDate date = LocalDate.now().minusDays(1);
+            Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
+            eventEntity.setDateCreated(Date.from(instant));
+            eventDao.createEvent(eventEntity);
+        }
+
+        for (int i = 0; i < 10; i++) {
+            EventEntity eventEntity = new EventEntity();
+            eventEntity.setEventType("Public-API");
+            eventEntity.setClientId("105.21.229.71");
+            eventEntity.setLabel("anonymous");
+            LocalDate date = LocalDate.now().minusDays(1);
+            Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
+            eventEntity.setDateCreated(Date.from(instant));
+            eventDao.createEvent(eventEntity);
+        }
+
+        for (int i = 0; i < 1100; i++) {
+            EventEntity eventEntity = new EventEntity();
+            eventEntity.setEventType("Public-API");
+            eventEntity.setClientId("104.20.228.70");
+            eventEntity.setLabel("anonymous");
+            LocalDate date = LocalDate.now().minusDays(1);
+            Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();
+            eventEntity.setDateCreated(Date.from(instant));
+            eventDao.createEvent(eventEntity);
+        }
+    }
+
+    private void createEvents() {
+        for (int i = 0; i < 40; i++) {
+            EventEntity eventEntity = new EventEntity();
+            eventEntity.setEventType(i % 2 == 0 ? "Sign-In" : "Public-PAPI");
             eventEntity.setClientId("Client " + 1);
             LocalDate date = LocalDate.now().minusDays(1);
             Instant instant = date.atStartOfDay(ZoneId.systemDefault()).toInstant();