feature: Start collection daily pub api stats (#7016)

* feature: Start collection daily pub api stats * fix: Delete event stats in tests * fix: Remove unused class * feature: Add ip table to event and event_stats --------- Co-authored-by: Daniel Palafox <[email protected]>
ORCID · Apr 4, 2024 · ceb7ce3 · ceb7ce3
1 parent e2208b0
commit ceb7ce3
Show file tree

Hide file tree

Showing 31 changed files with 414 additions and 445 deletions.
diff --git a/orcid-core/src/main/java/org/orcid/core/common/manager/EventManager.java b/orcid-core/src/main/java/org/orcid/core/common/manager/EventManager.java
@@ -2,7 +2,7 @@
 
 import javax.servlet.http.HttpServletRequest;
 
-import org.orcid.core.utils.EventType;
+import org.orcid.persistence.jpa.entities.EventType;
 
 /**
  *
@@ -13,4 +13,6 @@ public interface EventManager {
 
     void createEvent(EventType eventType, HttpServletRequest request);
 
+    void createPapiEvent(String clientId, String ip, boolean anonymous);
+
 }
diff --git a/orcid-core/src/main/java/org/orcid/core/common/manager/impl/EventManagerImpl.java b/orcid-core/src/main/java/org/orcid/core/common/manager/impl/EventManagerImpl.java
@@ -13,12 +13,12 @@
 import org.orcid.core.constants.OrcidOauth2Constants;
 import org.orcid.core.manager.ClientDetailsEntityCacheManager;
 import org.orcid.core.manager.v3.read_only.RecordNameManagerReadOnly;
-import org.orcid.core.utils.EventType;
 import org.orcid.jaxb.model.clientgroup.ClientType;
 import org.orcid.jaxb.model.v3.release.record.Name;
 import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.ClientDetailsEntity;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.pojo.ajaxForm.PojoUtil;
 import org.orcid.pojo.ajaxForm.RequestInfoForm;
 
@@ -105,4 +105,15 @@ private String removeAttributesFromUrl(String url) {
         }
         return url;
     }
+
+    @Override
+    public void createPapiEvent(String clientId, String ip, boolean anonymous) {
+        EventEntity eventEntity = new EventEntity();
+        eventEntity.setEventType(EventType.PAPI.getValue());
+        eventEntity.setClientId(clientId);
+        eventEntity.setIp(ip);
+        eventEntity.setLabel(anonymous ? "anonymous" : null);
+        eventEntity.setDateCreated(new Date());
+        eventDao.createEvent(eventEntity);
+    }
 }
diff --git a/orcid-core/src/main/java/org/orcid/core/togglz/Features.java b/orcid-core/src/main/java/org/orcid/core/togglz/Features.java
@@ -54,7 +54,10 @@ public enum Features implements Feature {
     SEND_ADD_WORKS_EMAILS,
 
     @Label("Delete events older than 90 days from the DB ")
-    DELETE_EVENTS;
+    DELETE_EVENTS,
+
+    @Label("Track public events stats ")
+    PAPI_EVENTS;
 
     public boolean isActive() {
         return FeatureContext.getFeatureManager().isActive(this);

diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java
@@ -1,6 +1,7 @@
 package org.orcid.persistence.dao;
 
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 
 import java.util.List;
 
@@ -18,6 +19,10 @@ public interface EventDao {
     void delete(long id);
 
     List<EventEntity> findAll();
+
+    List<EventEntity> findByEventType(EventType eventType);
 
     void deleteEventsByDate(Integer numberOfDays);
+
+    void deletePapiEvents(Integer numberOfDays);
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java
@@ -1,6 +1,7 @@
 package org.orcid.persistence.dao;
 
 import org.orcid.persistence.jpa.entities.EventStatsEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 
 import java.util.List;
 
@@ -13,5 +14,9 @@ public interface EventStatsDao {
 
     void createEventStats();
 
+    void createPapiEventStats();
+
     List<EventStatsEntity> findAll();
+
+    void deleteStatsByType(EventType eventType);
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java
@@ -7,6 +7,7 @@
 
 import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
@@ -49,9 +50,27 @@ public List<EventEntity> findAll() {
     @Override
     @Transactional
     public void deleteEventsByDate(Integer numberOfDays) {
-        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date)";
+        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type != :eventType";
         Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("eventType", EventType.PAPI.getValue());
         queryDelete.setParameter("numberOfDays", numberOfDays);
         queryDelete.executeUpdate();
     }
+
+    @Override
+    @Transactional
+    public void deletePapiEvents(Integer numberOfDays) {
+        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date) AND event_type = :eventType";
+        Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("eventType", EventType.PAPI.getValue());
+        queryDelete.setParameter("numberOfDays", numberOfDays);
+        queryDelete.executeUpdate();
+    }
+
+    @Override
+    public List<EventEntity> findByEventType(EventType eventType) {
+        TypedQuery<EventEntity> query = entityManager.createQuery("from EventEntity where eventType = :eventType", EventEntity.class);
+        query.setParameter("eventType", eventType.getValue());
+        return query.getResultList();
+    }
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java b/orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java
@@ -1,13 +1,17 @@
 package org.orcid.persistence.dao.impl;
 
 import org.orcid.persistence.dao.EventStatsDao;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.persistence.jpa.entities.EventStatsEntity;
 import org.springframework.transaction.annotation.Transactional;
 
 import javax.annotation.Resource;
 import javax.persistence.EntityManager;
 import javax.persistence.Query;
 import javax.persistence.TypedQuery;
+
+import java.math.BigInteger;
+import java.util.ArrayList;
 import java.util.List;
 
 /**
@@ -25,9 +29,11 @@ public void createEventStats() {
                 "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) " +
                 "SELECT event_type, client_id, COUNT(id), CAST(e.date_created as date), now(), now() " +
                 "FROM event as e " +
-                "WHERE CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
+                "WHERE event_type != '"+ EventType.PAPI.getValue() + "' " +
+                "AND CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
                 "GROUP BY event_type, client_id, CAST(e.date_created as date) " +
                 "ORDER BY CAST(e.date_created as date) DESC;";
+
         Query insertQuery = entityManager.createNativeQuery(query);
         insertQuery.executeUpdate();
     }
@@ -37,4 +43,49 @@ public List<EventStatsEntity> findAll() {
         TypedQuery<EventStatsEntity> query = entityManager.createQuery("from EventStatsEntity", EventStatsEntity.class);
         return query.getResultList();
     }
+
+    @Override
+    @Transactional
+    public void createPapiEventStats() {
+        String query = 
+                "SELECT event_type, client_id, ip, label, count(*), CAST(date_created as date), now(), now() " +
+                "FROM event " +
+                "WHERE event_type = '"+ EventType.PAPI.getValue() + "' " +
+                "AND CAST(date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
+                "GROUP BY event_type, client_id, ip, label, CAST(date_created as date) " +
+                "ORDER BY CAST(date_created as date) DESC;";
+
+        Query queryList = entityManager.createNativeQuery(query);
+        List<Object[]> eventsList = queryList.getResultList();
+        List<Object[]> eventsListToRemove = new ArrayList<>();
+        if (eventsList.size() > 0) {
+            eventsList.forEach(item -> {
+                if (item[3] == "anonymous" && item[4] != null && ((BigInteger) item[4]).intValue() < 1000) {
+                    eventsListToRemove.add(item);
+                }
+            });
+            eventsList.removeAll(eventsListToRemove);
+            eventsList.forEach(item -> {
+                String insertQuery = "INSERT INTO event_stats (event_type, client_id, ip, count, date, date_created, last_modified) VALUES (:eventType, :clientId, :ip, :count, :date, :dateCreated, :lastModified)";
+                Query insertQueryClients = entityManager.createNativeQuery(insertQuery);
+                insertQueryClients.setParameter("eventType", item[0]);
+                insertQueryClients.setParameter("clientId", item[1]);
+                insertQueryClients.setParameter("ip", item[2]);
+                insertQueryClients.setParameter("count", item[4]);
+                insertQueryClients.setParameter("date", item[5]);
+                insertQueryClients.setParameter("dateCreated", item[6]);
+                insertQueryClients.setParameter("lastModified", item[7]);
+                insertQueryClients.executeUpdate();
+            });
+        }
+    }
+
+    @Override
+    @Transactional
+    public void deleteStatsByType(EventType eventType) {
+        String query = "DELETE FROM event_stats where event_type = :eventType";
+        Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("eventType", eventType.getValue());
+        queryDelete.executeUpdate();
+    }
 }
diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventEntity.java
@@ -22,6 +22,7 @@ public class EventEntity {
     private static final long serialVersionUID = 1L;
     private Long id;
     private String clientId;
+    private String ip;
     private String eventType;
     private String label;
     private Date dateCreated;
@@ -56,6 +57,15 @@ public void setClientId(String client_id) {
         this.clientId = client_id;
     }
 
+    @Column(name = "ip")
+    public String getIp() {
+        return ip;
+    }
+
+    public void setIp(String ip) {
+        this.ip = ip;
+    }
+
     @Column(name = "label")
     public String getLabel() {
         return label;

diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java
@@ -22,6 +22,7 @@ public class EventStatsEntity extends BaseEntity<Long>{
     private Long id;
     private String eventType;
     private String clientId;
+    private String ip;
     private Integer count;
     private Date date;
 
@@ -55,6 +56,15 @@ public void setClientId(String client_id) {
         this.clientId = client_id;
     }
 
+    @Column(name = "ip")
+    public String getIp() {
+        return ip;
+    }
+
+    public void setIp(String ip) {
+        this.ip = ip;
+    }
+
     @Column(name = "count")
     public Integer getCount() {
         return count;

diff --git a/.../java/org/orcid/core/utils/EventType.java → ...d/persistence/jpa/entities/EventType.java b/.../java/org/orcid/core/utils/EventType.java → ...d/persistence/jpa/entities/EventType.java
@@ -1,12 +1,13 @@
-package org.orcid.core.utils;
+package org.orcid.persistence.jpa.entities;
 
 public enum EventType {
     SIGN_IN("Sign-In"),
     NEW_REGISTRATION("New-Registration"),
     AUTHORIZE("Authorize"),
     AUTHORIZE_DENY("Authorize-Deny"),
     REAUTHORIZE("Reauthorize"),
-    PUBLIC_PAGE("Public-Page");
+    PUBLIC_PAGE("Public-Page"),
+    PAPI("Public-API");
 
     private final String value;
 
@@ -17,4 +18,4 @@ public enum EventType {
     public String getValue() {
         return value;
     }
-}
+}
diff --git a/orcid-persistence/src/main/resources/db-master.xml b/orcid-persistence/src/main/resources/db-master.xml
@@ -385,4 +385,6 @@
   <include file="/db/updates/dw_event_stats.xml" />  
   <include file="/db/updates/add_indexes_2024_02_20.xml" />
   <include file="/db/updates/dw_views_with_4_months_interval.xml" />
+  <include file="/db/updates/dw_papi_event_stats.xml" />
+  <include file="/db/updates/add_ip_col_in_events.xml" />
 </databaseChangeLog>
diff --git a/orcid-persistence/src/main/resources/db/updates/add_ip_col_in_events.xml b/orcid-persistence/src/main/resources/db/updates/add_ip_col_in_events.xml
@@ -0,0 +1,23 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+	xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">	
+
+    <changeSet id="ADD-FIELD-TO-EVENT" author="Daniel Palafox">
+    	<preConditions onFail="MARK_RAN">
+			<not>
+				<columnExists tableName="event" columnName="ip"/>
+			</not>
+		</preConditions>
+		<sql>ALTER TABLE event ADD ip VARCHAR(15);</sql>
+    </changeSet>
+
+    <changeSet id="ADD-FIELD-TO-EVENT_STATS" author="Daniel Palafox">
+    	<preConditions onFail="MARK_RAN">
+			<not>
+				<columnExists tableName="event_stats" columnName="ip"/>
+			</not>
+		</preConditions>
+		<sql>ALTER TABLE event_stats ADD ip VARCHAR(15);</sql>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/orcid-persistence/src/main/resources/db/updates/dw_papi_event_stats.xml b/orcid-persistence/src/main/resources/db/updates/dw_papi_event_stats.xml
@@ -0,0 +1,45 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+    <changeSet id="DROP-VIEW-FOR-EVENT" author="Daniel Palafox" dbms="postgresql">
+        <dropView viewName="dw_event_stats"/>
+    </changeSet>
+
+    <changeSet id="CREATE-DW-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions onFail="MARK_RAN">
+            <not><viewExists viewName="dw_event_stats"/></not>
+        </preConditions>
+        <createView viewName="dw_event_stats">
+            SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
+            FROM event_stats WHERE event_type != 'Public-API'
+            ORDER BY DATE_TRUNC('day', date_created) DESC;
+        </createView>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-TO-DW_USER-TO-DW_EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON TABLE dw_event_stats to dw_user;</sql>
+    </changeSet>
+
+    <changeSet id="CREATE-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions onFail="MARK_RAN">
+            <not><viewExists viewName="dw_papi_event_stats"/></not>
+        </preConditions>
+        <createView viewName="dw_papi_event_stats">
+            SELECT event_type, client_id, ip, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
+            FROM event_stats WHERE event_type = 'Public-API'
+            ORDER BY DATE_TRUNC('day', date_created) DESC;
+        </createView>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-TO-DW-USER-TO-DW-PAPI-EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON TABLE dw_papi_event_stats to dw_user;</sql>
+    </changeSet>
+
+</databaseChangeLog>
diff --git a/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java b/orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java
@@ -17,6 +17,7 @@
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.orcid.persistence.jpa.entities.EventEntity;
+import org.orcid.persistence.jpa.entities.EventType;
 import org.orcid.test.DBUnitTest;
 import org.orcid.test.OrcidJUnit4ClassRunner;
 import org.springframework.test.context.ContextConfiguration;
@@ -81,4 +82,19 @@ public void testWriteEvent() throws IllegalAccessException {
 
         eventDao.delete(eventEntity.getId());
     }
+
+    @Test
+    public void deletePapiEventsByDate() {
+        List<EventEntity> eventEntityList = eventDao.findByEventType(EventType.PAPI);
+
+        assertNotNull(eventEntityList);
+        assertEquals(3, eventEntityList.size());
+
+        eventDao.deletePapiEvents(90);
+
+        eventEntityList = eventDao.findByEventType(EventType.PAPI);
+
+        assertNotNull(eventEntityList);
+        assertEquals(0, eventEntityList.size());
+    }
 }