Grad release 1.12.0

api/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>ca.bc.gov.educ</groupId>
     <artifactId>educ-grad-batch-graduation-api</artifactId>
-    <version>1.8.54</version>
+    <version>1.8.55</version>
     <name>educ-grad-batch-graduation-api</name>
     <description>Ministry of Education GRAD BATCH GRADUATION API</description>
 

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/config/BatchJobLauncher.java

@@ -146,4 +146,17 @@ public void refreshUserScheduledQueue() {
 
         LOGGER.info(BATCH_ENDED);
     }
+
+    @Scheduled(cron = "${batch.purge-old-records.cron}")
+    @SchedulerLock(name = "PurgeOldRecordsLock",
+            lockAtLeastFor = "PT1H", lockAtMostFor = "PT1H") //midnight job so lock for an hour
+    public void purgeOldRecords() {
+        LockAssert.assertLocked();
+        try {
+            this.gradDashboardService.purgeOldBatchHistoryRecords();
+            this.gradDashboardService.purgeOldSpringMetaDataRecords();
+        } catch (Exception e) {
+            LOGGER.error(ERROR_MSG, e.getLocalizedMessage());
+        }
+    }
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchJobExecutionContextEntity.java

@@ -11,7 +11,7 @@
 public class BatchJobExecutionContextEntity {
     @Id
     @Column(name = "JOB_EXECUTION_ID", nullable = false)
-    private Long id;
+    private Long jobExecutionId;
 
     @Column(name = "SHORT_CONTEXT", nullable = false, length = 2500)
     private String shortContext;

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionContextEntity.java

@@ -10,7 +10,7 @@ public class BatchStepExecutionContextEntity {
 
     @Id
     @Column(name = "STEP_EXECUTION_ID", nullable = false)
-    private Long id;
+    private Long stepExecutionId;
 
     @Column(name = "SHORT_CONTEXT", length = 2500)
     private String shortContext;

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/entity/BatchStepExecutionEntity.java

@@ -12,7 +12,7 @@ public class BatchStepExecutionEntity {
 
     @Id
     @Column(name = "STEP_EXECUTION_ID", nullable = false)
-    private Long id;
+    private Long stepExecutionId;
 
     @Column(name = "JOB_EXECUTION_ID", nullable = false)
     private Long jobExecutionId;

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmJobHistoryRepository.java

@@ -1,13 +1,17 @@
 package ca.bc.gov.educ.api.batchgraduation.repository;
 
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
 
 import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity;
+import org.springframework.transaction.annotation.Transactional;
 
 @Repository
 public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository<BatchGradAlgorithmJobHistoryEntity, UUID> {
@@ -16,4 +20,8 @@ public interface BatchGradAlgorithmJobHistoryRepository extends JpaRepository<Ba
 
     Optional<BatchGradAlgorithmJobHistoryEntity> findByJobExecutionId(Long batchId);
 
+    @Transactional
+    @Modifying
+    @Query("delete from BatchGradAlgorithmJobHistoryEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchGradAlgorithmStudentRepository.java

@@ -8,7 +8,9 @@
 import org.springframework.data.jpa.repository.Query;
 import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.time.LocalDateTime;
 import java.util.*;
 
 @Repository
@@ -58,5 +60,10 @@ void copyAllGradAlgorithmStudents(
             "and status = 'COMPLETED'\n" +
             "group by graduation_program_code\n" +
             "order by graduation_program_code desc", nativeQuery = true)
-    List<Object[]> getGraduationProgramCounts( @Param("batchId") Long batchId);
+    List<Object[]> getGraduationProgramCounts(@Param("batchId") Long batchId);
+
+    @Transactional
+    @Modifying
+    @Query("delete from BatchGradAlgorithmStudentEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchJobExecutionRepository.java

@@ -4,11 +4,38 @@
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.time.LocalDateTime;
 
 @Repository
 public interface BatchJobExecutionRepository extends JpaRepository<BatchJobExecutionEntity, Long> {
 
     Page<BatchJobExecutionEntity> findAllByOrderByCreateTimeDesc(Pageable page);
 
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchJobExecutionParamEntity bjep WHERE bjep.jobExecutionId IN (\n" +
+            "SELECT bje.jobExecutionId FROM BatchJobExecutionEntity bje WHERE bje.createTime <= :createDate)")
+    void deleteBatchParamsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchJobExecutionContextEntity bjec WHERE bjec.jobExecutionId IN (\n" +
+            "SELECT bje.jobExecutionId FROM BatchJobExecutionEntity bje WHERE bje.createTime <= :createDate)")
+    void deleteBatchContextsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchJobInstanceEntity bji WHERE bji.id NOT IN (\n" +
+            "SELECT bje.id FROM BatchJobExecutionEntity bje)")
+    void deleteBatchInstancesNotInBatchJobs();
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchJobExecutionEntity bje WHERE bje.createTime <= :createDate")
+    void deleteBatchJobsByCreateTimeBefore(LocalDateTime createDate);
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/BatchStepExecutionRepository.java

@@ -2,13 +2,30 @@
 
 import ca.bc.gov.educ.api.batchgraduation.entity.BatchStepExecutionEntity;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
+import org.springframework.data.jpa.repository.Query;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
+import java.time.LocalDateTime;
 import java.util.List;
 
 @Repository
 public interface BatchStepExecutionRepository extends JpaRepository<BatchStepExecutionEntity, Long> {
 
     List<BatchStepExecutionEntity> findByJobExecutionIdOrderByEndTimeDesc(Long jobExecutionId);
 
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchStepExecutionContextEntity bsec WHERE bsec.stepExecutionId IN (\n" +
+            "SELECT bse.stepExecutionId FROM BatchStepExecutionEntity bse WHERE bse.jobExecutionId IN (\n" +
+            "    SELECT bje.jobExecutionId FROM BatchJobExecutionEntity bje WHERE bje.createTime <= :createDate))")
+    void deleteBatchStepContextsByCreateTimeBefore(LocalDateTime createDate);
+
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM BatchStepExecutionEntity bse WHERE bse.jobExecutionId IN (\n" +
+            "    SELECT bje.jobExecutionId FROM BatchJobExecutionEntity bje WHERE bje.createTime <= :createDate)")
+    void deleteBatchStepsByCreateTimeBefore(LocalDateTime createDate);
+
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/repository/StudentCredentialDistributionRepository.java

@@ -1,16 +1,15 @@
 package ca.bc.gov.educ.api.batchgraduation.repository;
 
-import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmStudentEntity;
 import ca.bc.gov.educ.api.batchgraduation.entity.StudentCredentialDistributionEntity;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
 import org.springframework.data.jpa.repository.JpaRepository;
 import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
-import org.springframework.data.repository.query.Param;
 import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Transactional;
 
-import java.util.Date;
+import java.time.LocalDateTime;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -37,4 +36,9 @@ public interface StudentCredentialDistributionRepository extends JpaRepository<S
 
     long countAllByJobExecutionId(Long batchId);
 
+    @Transactional
+    @Modifying
+    @Query("delete from StudentCredentialDistributionEntity where createDate <= :createDate")
+    void deleteByCreateDateBefore(LocalDateTime createDate);
+
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardService.java

@@ -6,6 +6,7 @@
 import ca.bc.gov.educ.api.batchgraduation.rest.RestUtils;
 import ca.bc.gov.educ.api.batchgraduation.transformer.BatchGradAlgorithmJobHistoryTransformer;
 import ca.bc.gov.educ.api.batchgraduation.transformer.BatchProcessingTransformer;
+import ca.bc.gov.educ.api.batchgraduation.util.EducGradBatchGraduationApiConstants;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.PageRequest;
 import org.springframework.data.domain.Pageable;
@@ -29,23 +30,30 @@ public class GradDashboardService extends GradService {
 	private final BatchStepExecutionRepository batchStepExecutionRepository;
 	private final BatchProcessingTransformer batchProcessingTransformer;
 	private final BatchProcessingRepository batchProcessingRepository;
+	private final StudentCredentialDistributionRepository studentCredentialDistributionRepository;
 	private final RestUtils restUtils;
 
+	private final EducGradBatchGraduationApiConstants constants;
+
     public GradDashboardService(BatchGradAlgorithmJobHistoryRepository batchGradAlgorithmJobHistoryRepository,
 								BatchGradAlgorithmJobHistoryTransformer batchGradAlgorithmJobHistoryTransformer,
 								RestUtils restUtils,
 								BatchJobExecutionRepository batchJobExecutionRepository,
 								BatchStepExecutionRepository batchStepExecutionRepository,
 								BatchProcessingRepository batchProcessingRepository,BatchProcessingTransformer batchProcessingTransformer,
-								BatchGradAlgorithmStudentRepository batchGradAlgorithmStudentRepository) {
+								BatchGradAlgorithmStudentRepository batchGradAlgorithmStudentRepository,
+								StudentCredentialDistributionRepository studentCredentialDistributionRepository,
+								EducGradBatchGraduationApiConstants constants) {
 		this.batchGradAlgorithmJobHistoryRepository = batchGradAlgorithmJobHistoryRepository;
 		this.batchGradAlgorithmJobHistoryTransformer = batchGradAlgorithmJobHistoryTransformer;
 		this.batchProcessingTransformer = batchProcessingTransformer;
 		this.batchGradAlgorithmStudentRepository = batchGradAlgorithmStudentRepository;
 		this.batchJobExecutionRepository = batchJobExecutionRepository;
 		this.batchStepExecutionRepository = batchStepExecutionRepository;
 		this.batchProcessingRepository = batchProcessingRepository;
+		this.studentCredentialDistributionRepository = studentCredentialDistributionRepository;
 		this.restUtils = restUtils;
+		this.constants = constants;
 	}
 
     @Transactional(readOnly = true)
@@ -198,4 +206,30 @@ private void updateBatchJobStatus(BatchGradAlgorithmJobHistory batchJobHistory,
 		batchGradAlgorithmJobHistoryRepository.save(entity);
 		batchJobHistory.setStatus(BatchStatusEnum.FAILED.toString());
 	}
+
+	@Transactional
+	public void purgeOldBatchHistoryRecords() {
+		final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
+		this.batchGradAlgorithmStudentRepository.deleteByCreateDateBefore(createDateToCompare);
+		this.studentCredentialDistributionRepository.deleteByCreateDateBefore(createDateToCompare);
+		this.batchGradAlgorithmJobHistoryRepository.deleteByCreateDateBefore(createDateToCompare);
+	}
+
+	@Transactional
+	public void purgeOldSpringMetaDataRecords() {
+		final LocalDateTime createDateToCompare = this.calculateCreateDateBasedOnStaleEventInDays();
+		this.batchStepExecutionRepository.deleteBatchStepContextsByCreateTimeBefore(createDateToCompare);
+		this.batchStepExecutionRepository.deleteBatchStepsByCreateTimeBefore(createDateToCompare);
+
+		this.batchJobExecutionRepository.deleteBatchParamsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchContextsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchJobsByCreateTimeBefore(createDateToCompare);
+		this.batchJobExecutionRepository.deleteBatchInstancesNotInBatchJobs();
+	}
+
+
+	private LocalDateTime calculateCreateDateBasedOnStaleEventInDays() {
+		final LocalDateTime currentTime = LocalDateTime.now();
+		return currentTime.minusDays(this.constants.getRecordsStaleInDays());
+	}
 }

api/src/main/java/ca/bc/gov/educ/api/batchgraduation/util/EducGradBatchGraduationApiConstants.java

@@ -273,4 +273,7 @@ public class EducGradBatchGraduationApiConstants {
     @Value("${splunk.log-helper.enabled}")
     private boolean splunkLogHelperEnabled;
 
+    @Value("${batch.purge-old-records.staleInDays}")
+    private int recordsStaleInDays;
+
 }

api/src/main/resources/application.yaml

@@ -158,6 +158,9 @@ batch:
       routines:
         lockAtLeastFor: ${CRON_SYSTEM_SCHEDULED_ROUTINES_LOCK_AT_LEAST_FOR}
         lockAtMostFor: ${CRON_SYSTEM_SCHEDULED_ROUTINES_LOCK_AT_MOST_FOR}
+  purge-old-records:
+    cron: ${CRON_SCHEDULED_PURGE_OLD_RECORDS}
+    staleInDays: ${RECORDS_STALE_IN_DAYS}
 
 #Endpoints
 endpoint:

api/src/main/resources/db/migration/1.0/V1.0.44__DDL-DROP_TABLE-batch_job_execution_params_old.sql

@@ -0,0 +1 @@
+DROP TABLE API_GRAD_BATCH.BATCH_JOB_EXECUTION_PARAMS_OLD;

api/src/test/java/ca/bc/gov/educ/api/batchgraduation/service/GradDashboardServiceTest.java

@@ -129,7 +129,7 @@ public void testGetDashboardInfo_whenLastUpdatedDate_isOlderThan5hours_thenUpdat
         BatchStepExecutionEntity step = new BatchStepExecutionEntity();
         step.setStepName("test-partition12");
         step.setJobExecutionId(hist.getJobExecutionId());
-        step.setId(Long.valueOf("123"));
+        step.setStepExecutionId(Long.valueOf("123"));
         step.setStatus("STARTED");
         step.setStartTime(ca.bc.gov.educ.api.batchgraduation.util.DateUtils.toLocalDateTime(startedDateTime));
         Date lastUpdatedDateTime = DateUtils.addHours(today, -6);
@@ -228,4 +228,26 @@ public void testGetBatchSummary() {
         assertThat(res.getBatchJobList()).hasSize(1);
     }
 
+    @Test
+    public void testPurgeOldBatchHistoryRecords() {
+        boolean isExceptionThrown = false;
+        try {
+            gradDashboardService.purgeOldBatchHistoryRecords();
+        } catch (Exception e) {
+            isExceptionThrown = true;
+        }
+        assertThat(isExceptionThrown).isFalse();
+    }
+
+    @Test
+    public void testPurgeOldSpringMetaDataRecords() {
+        boolean isExceptionThrown = false;
+        try {
+            gradDashboardService.purgeOldSpringMetaDataRecords();
+        } catch (Exception e) {
+            isExceptionThrown = true;
+        }
+        assertThat(isExceptionThrown).isFalse();
+    }
+
 }

api/src/test/resources/application.yaml

@@ -87,6 +87,9 @@ batch:
       routines:
         lockAtLeastFor: PT1M
         lockAtMostFor: PT600M
+  purge-old-records:
+    cron: 0 30 0 * * *
+    staleInDays: 90
 
 #Endpoints
 endpoint: