Merge pull request #6960 from ORCID/8983-tech-cron-job-to-store-the-d…

…aily-stats-in-eventstats-and-remove-old-events

feature: Add cron job to delete events and create events stats

orcid-core/src/main/java/org/orcid/core/togglz/Features.java

@@ -40,7 +40,10 @@ public enum Features implements Feature {
     SEND_ALL_VERIFICATION_EMAILS,
 
     @Label("Send add works emails for 7, 28 and 90 days.")
-    SEND_ADD_WORKS_EMAILS;
+    SEND_ADD_WORKS_EMAILS,
+
+    @Label("Delete events older than 90 days from the DB ")
+    DELETE_EVENTS;
 
     public boolean isActive() {
         return FeatureContext.getFeatureManager().isActive(this);

orcid-persistence/src/main/java/org/orcid/persistence/dao/EventDao.java

@@ -2,6 +2,8 @@
 
 import org.orcid.persistence.jpa.entities.EventEntity;
 
+import java.util.List;
+
 /**
  *
  * @author Daniel Palafox
@@ -12,4 +14,10 @@ public interface EventDao {
     void createEvent(EventEntity eventEntity);
 
     EventEntity find(long id);
+
+    void delete(long id);
+
+    List<EventEntity> findAll();
+
+    void deleteEventsByDate(Integer numberOfDays);
 }

orcid-persistence/src/main/java/org/orcid/persistence/dao/EventStatsDao.java

@@ -0,0 +1,17 @@
+package org.orcid.persistence.dao;
+
+import org.orcid.persistence.jpa.entities.EventStatsEntity;
+
+import java.util.List;
+
+/**
+ *
+ * @author Daniel Palafox
+ *
+ */
+public interface EventStatsDao {
+
+    void createEventStats();
+
+    List<EventStatsEntity> findAll();
+}

orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventDaoImpl.java

@@ -2,11 +2,15 @@
 
 import javax.annotation.Resource;
 import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
 
 import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.EventEntity;
 import org.springframework.transaction.annotation.Transactional;
 
+import java.util.List;
+
 /**
  * @author Daniel Palafox
  */
@@ -29,5 +33,25 @@ public void createEvent(EventEntity eventEntity) {
     public EventEntity find(long id) {
         return entityManager.find(EventEntity.class, id);
     }
-
+
+    @Override
+    @Transactional
+    public void delete(long id) {
+        entityManager.remove(find(id));
+    }
+
+    @Override
+    public List<EventEntity> findAll() {
+        TypedQuery<EventEntity> query = entityManager.createQuery("from EventEntity", EventEntity.class);
+        return query.getResultList();
+    }
+
+    @Override
+    @Transactional
+    public void deleteEventsByDate(Integer numberOfDays) {
+        String query = "DELETE FROM event where CAST(date_created as date) < CAST(now() - (CAST('1' AS INTERVAL DAY) * :numberOfDays) as date)";
+        Query queryDelete = entityManager.createNativeQuery(query);
+        queryDelete.setParameter("numberOfDays", numberOfDays);
+        queryDelete.executeUpdate();
+    }
 }

orcid-persistence/src/main/java/org/orcid/persistence/dao/impl/EventStatsDaoImpl.java

@@ -0,0 +1,40 @@
+package org.orcid.persistence.dao.impl;
+
+import org.orcid.persistence.dao.EventStatsDao;
+import org.orcid.persistence.jpa.entities.EventStatsEntity;
+import org.springframework.transaction.annotation.Transactional;
+
+import javax.annotation.Resource;
+import javax.persistence.EntityManager;
+import javax.persistence.Query;
+import javax.persistence.TypedQuery;
+import java.util.List;
+
+/**
+ * @author Daniel Palafox
+ */
+public class EventStatsDaoImpl implements EventStatsDao {
+
+    @Resource(name = "entityManager")
+    protected EntityManager entityManager;
+
+    @Override
+    @Transactional
+    public void createEventStats() {
+        String query = 
+                "INSERT INTO event_stats (event_type, client_id, count, date, date_created, last_modified) " +
+                "SELECT event_type, client_id, COUNT(id), CAST(e.date_created as date), now(), now() " +
+                "FROM event as e " +
+                "WHERE CAST(e.date_created as date) = CAST(now() - (CAST('1' AS INTERVAL DAY) * 1) as date) " +
+                "GROUP BY event_type, client_id, CAST(e.date_created as date) " +
+                "ORDER BY CAST(e.date_created as date) DESC;";
+        Query insertQuery = entityManager.createNativeQuery(query);
+        insertQuery.executeUpdate();
+    }
+
+    @Override
+    public List<EventStatsEntity> findAll() {
+        TypedQuery<EventStatsEntity> query = entityManager.createQuery("from EventStatsEntity", EventStatsEntity.class);
+        return query.getResultList();
+    }
+}

orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/EventStatsEntity.java

@@ -0,0 +1,92 @@
+package org.orcid.persistence.jpa.entities;
+
+import java.util.Date;
+import java.util.Objects;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.GenerationType;
+import javax.persistence.Id;
+import javax.persistence.SequenceGenerator;
+import javax.persistence.Table;
+
+/**
+ *
+ * @author Daniel Palafox
+ *
+ */
+@Entity
+@Table(name = "event_stats")
+public class EventStatsEntity extends BaseEntity<Long>{
+    private Long id;
+    private String eventType;
+    private String clientId;
+    private Integer count;
+    private Date date;
+
+    @Id
+    @Column(name = "id")
+    @GeneratedValue(strategy = GenerationType.AUTO, generator = "event_seq")
+    @SequenceGenerator(name = "event_seq", sequenceName = "event_seq", allocationSize = 1)
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    @Column(name = "event_type")
+    public String getEventType() {
+        return eventType;
+    }
+
+    public void setEventType(String eventType) {
+        this.eventType = eventType;
+    }
+
+    @Column(name = "client_id")
+    public String getClientId() {
+        return clientId;
+    }
+
+    public void setClientId(String client_id) {
+        this.clientId = client_id;
+    }
+
+    @Column(name = "count")
+    public Integer getCount() {
+        return count;
+    }
+
+    public void setCount(Integer count) {
+        this.count = count;
+    }
+
+    @Column(name = "date")
+    public Date getDate() {
+        return date;
+    }
+
+    public void setDate(Date date) {
+        this.date = date;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        EventStatsEntity that = (EventStatsEntity) o;
+        return Objects.equals(getId(), that.getId()) &&
+                Objects.equals(getEventType(), that.getEventType()) &&
+                Objects.equals(getClientId(), that.getClientId()) &&
+                Objects.equals(getCount(), that.getCount()) &&
+                Objects.equals(getDate(), that.getDate());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(getId(), getEventType(), getClientId(), getCount(), getDate());
+    }
+}

orcid-persistence/src/main/resources/META-INF/persistence.xml

@@ -61,6 +61,7 @@
         <class>org.orcid.persistence.jpa.entities.FindMyStuffHistoryEntity</class>
         <class>org.orcid.persistence.jpa.entities.SpamEntity</class>
         <class>org.orcid.persistence.jpa.entities.EventEntity</class>
+        <class>org.orcid.persistence.jpa.entities.EventStatsEntity</class>
         <class>org.orcid.persistence.jpa.entities.EmailDomainEntity</class>
 
         <!-- OAuth entities -->

orcid-persistence/src/main/resources/db-master.xml

@@ -381,4 +381,6 @@
   <include file="/db/updates/dw_alter_event_2.xml" />
   <include file="/db/updates/grant_select_to_dw_user_on_dw_event.xml" />
   <include file="/db/updates/identifier-types/update-ol-to-be-case-sensitive.xml" />
+  <include file="/db/updates/create_event_stats_table.xml" />
+  <include file="/db/updates/dw_event_stats.xml" />
 </databaseChangeLog>

orcid-persistence/src/main/resources/db/updates/create_event_stats_table.xml

@@ -0,0 +1,40 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+    <changeSet id="CREATE-EVENT-STATS-TABLE" author="Daniel Palafox">
+        <createTable tableName="event_stats">
+            <column name="id" type="bigint" autoIncrement="true">
+                <constraints nullable="false" primaryKey="true"
+                             primaryKeyName="event_stats_pkey"/>
+            </column>
+            <column name="event_type" type="VARCHAR(20)"/>
+            <column name="client_id" type="VARCHAR(255)"/>
+            <column name="count" type="bigint"/>
+            <column name="date" type="TIMESTAMP"/>
+            <column name="date_created" type="TIMESTAMP WITH TIME ZONE"/>
+            <column name="last_modified" type="TIMESTAMP WITH TIME ZONE"/>
+        </createTable>
+    </changeSet>
+
+    <changeSet id="CREATE-EVENT-STATS-SEQUENCES" author="Daniel Palafox">
+        <preConditions onFail="MARK_RAN">
+            <not>
+                <sequenceExists sequenceName="event_stats_seq"/>
+            </not>
+        </preConditions>
+        <createSequence sequenceName="event_stats_seq"/>
+    </changeSet>
+
+    <changeSet id="CREATE-EVENT-STATS-AUTOCOLS" author="Daniel Palafox" dbms="hsqldb">
+        <addAutoIncrement tableName="event_stats" columnName="id" columnDataType="bigint"/>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-PERMISSIONS-TO-ORCIDRO-ON-EVENT-STATS" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='orcidro'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON event_stats to orcidro;</sql>
+    </changeSet>
+
+</databaseChangeLog>

orcid-persistence/src/main/resources/db/updates/dw_event_stats.xml

@@ -0,0 +1,27 @@
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-2.0.xsd">
+
+    <changeSet id="DROP-VIEW-FOR-EVENT" author="Daniel Palafox" dbms="postgresql">
+        <dropView viewName="dw_event"/>
+    </changeSet>
+
+    <changeSet id="CREATE-DW-EVENT-STATS-VIEW-GROUP-BY-DAY-CLIENT_ID-AND-EVENT_TYPE" author="Daniel Palafox" dbms="postgresql">
+        <preConditions onFail="MARK_RAN">
+            <not><viewExists viewName="dw_event_stats"/></not>
+        </preConditions>
+        <createView viewName="dw_event_stats">
+            SELECT event_type, client_id, count, DATE_TRUNC('day', date), DATE_TRUNC('day', date) as last_modified
+            FROM event_stats
+            ORDER BY DATE_TRUNC('day', date_created) DESC;
+        </createView>
+    </changeSet>
+
+    <changeSet id="GRANT-READ-TO-DW_USER-TO-DW_EVENT-STATS-VIEW" author="Daniel Palafox" dbms="postgresql">
+        <preConditions>
+            <sqlCheck expectedResult="1">SELECT 1 FROM pg_roles WHERE rolname='dw_user'</sqlCheck>
+        </preConditions>
+        <sql>GRANT SELECT ON TABLE dw_event_stats to dw_user;</sql>
+    </changeSet>
+
+</databaseChangeLog>

orcid-persistence/src/main/resources/orcid-persistence-context.xml

@@ -459,7 +459,8 @@
 
     <bean id="eventDao" class="org.orcid.persistence.dao.impl.EventDaoImpl" />
 
-
+    <bean id="eventStatsDao" class="org.orcid.persistence.dao.impl.EventStatsDaoImpl" />
+
     <bean id="emailDomainDao" class="org.orcid.persistence.dao.impl.EmailDomainDaoImpl" />
 
     <bean id="emailDomainDaoReadOnly" class="org.orcid.persistence.dao.impl.EmailDomainDaoImpl">

orcid-persistence/src/test/java/org/orcid/persistence/EventDaoTest.java → orcid-persistence/src/test/java/org/orcid/persistence/dao/EventDaoTest.java

@@ -1,20 +1,21 @@
-package org.orcid.persistence;
+package org.orcid.persistence.dao;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 
 import java.util.Arrays;
 import java.util.Date;
+import java.util.List;
 
 import javax.annotation.Resource;
+import javax.persistence.EntityManager;
 
 import org.apache.commons.lang3.reflect.FieldUtils;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 import org.junit.runner.RunWith;
-import org.orcid.persistence.dao.EventDao;
 import org.orcid.persistence.jpa.entities.EventEntity;
 import org.orcid.test.DBUnitTest;
 import org.orcid.test.OrcidJUnit4ClassRunner;
@@ -39,6 +40,21 @@ public static void removeDBUnitData() throws Exception {
         removeDBUnitData(Arrays.asList("/data/EventEntityData.xml", "/data/ProfileEntityData.xml", "/data/SourceClientDetailsEntityData.xml"));
     }
 
+    @Test
+    public void deleteEventsByDate() {
+        List<EventEntity> eventEntityList = eventDao.findAll();
+
+        assertNotNull(eventEntityList);
+        assertEquals(3, eventEntityList.size());
+
+        eventDao.deleteEventsByDate(90);
+
+        eventEntityList = eventDao.findAll();
+
+        assertNotNull(eventEntityList);
+        assertEquals(0, eventEntityList.size());
+    }
+
     @Test
     public void testWriteEvent() throws IllegalAccessException {
         EventEntity eventEntity = new EventEntity();
@@ -62,6 +78,7 @@ public void testWriteEvent() throws IllegalAccessException {
         assertEquals(eventEntity.getId(), fromDb.getId());
         assertEquals(eventEntity.getLabel(), fromDb.getLabel());
         assertNotNull(fromDb.getDateCreated());
+
+        eventDao.delete(eventEntity.getId());
     }
-
 }