From 4fc3df08304db9d5c4b25c6571526897ba95de92 Mon Sep 17 00:00:00 2001 From: Noah Overcash Date: Tue, 6 Feb 2024 17:08:47 -0500 Subject: [PATCH] Simplify migration logic --- folio-export-common | 2 +- .../folio/des/service/FolioTenantService.java | 26 +- .../BursarExportLegacyJobService.java | 44 +-- .../bursarlegacy/BursarMigrationService.java | 333 ++++++------------ .../folio/des/service/util/JobMapperUtil.java | 41 ++- src/main/resources/swagger.api/jobs.yaml | 2 - .../des/service/FolioTenantServiceTest.java | 23 +- .../BursarExportLegacyJobServiceTest.java | 98 ++++-- .../BursarMigrationServiceTest.java | 264 +++++--------- 9 files changed, 342 insertions(+), 491 deletions(-) diff --git a/folio-export-common b/folio-export-common index d572f3df..7083d609 160000 --- a/folio-export-common +++ b/folio-export-common @@ -1 +1 @@ -Subproject commit d572f3df0aa05aaa1ddbef0808e1c1683895b99b +Subproject commit 7083d609fab70b123fb292d9934ed4f5f0bff117 diff --git a/src/main/java/org/folio/des/service/FolioTenantService.java b/src/main/java/org/folio/des/service/FolioTenantService.java index 9bc67791..93acec82 100644 --- a/src/main/java/org/folio/des/service/FolioTenantService.java +++ b/src/main/java/org/folio/des/service/FolioTenantService.java @@ -34,12 +34,21 @@ public class FolioTenantService extends TenantService { private final PrepareSystemUserService prepareSystemUserService; private final BursarMigrationService bursarMigrationService; - public FolioTenantService(JdbcTemplate jdbcTemplate, FolioExecutionContext context, FolioSpringLiquibase folioSpringLiquibase, - PrepareSystemUserService prepareSystemUserService, KafkaService kafka, - BulkEditConfigService bulkEditConfigService, EdifactScheduledJobInitializer edifactScheduledJobInitializer, - ScheduledJobsRemover scheduledJobsRemover, BursarScheduledJobInitializer bursarScheduledJobInitializer, - OldJobDeleteScheduler oldJobDeleteScheduler, BursarExportLegacyJobService bursarExportLegacyJobService, - JobService jobService, BursarMigrationService bursarMigrationService) { + public FolioTenantService( + JdbcTemplate jdbcTemplate, + FolioExecutionContext context, + FolioSpringLiquibase folioSpringLiquibase, + PrepareSystemUserService prepareSystemUserService, + KafkaService kafka, + BulkEditConfigService bulkEditConfigService, + EdifactScheduledJobInitializer edifactScheduledJobInitializer, + ScheduledJobsRemover scheduledJobsRemover, + BursarScheduledJobInitializer bursarScheduledJobInitializer, + OldJobDeleteScheduler oldJobDeleteScheduler, + BursarExportLegacyJobService bursarExportLegacyJobService, + JobService jobService, + BursarMigrationService bursarMigrationService + ) { super(jdbcTemplate, context, folioSpringLiquibase); this.prepareSystemUserService = prepareSystemUserService; this.kafka = kafka; @@ -63,7 +72,10 @@ protected void afterTenantUpdate(TenantAttributes tenantAttributes) { oldJobDeleteScheduler.scheduleOldJobDeletion(context.getTenantId()); kafka.createKafkaTopics(); kafka.restartEventListeners(); - bursarMigrationService.recreateLegacyJobs(bursarExportLegacyJobService, jobService); + bursarMigrationService.updateLegacyBursarJobs( + bursarExportLegacyJobService, + jobService + ); } catch (Exception e) { log.error(e.getMessage(), e); throw e; diff --git a/src/main/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobService.java b/src/main/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobService.java index d150554c..f8c2ea5d 100644 --- a/src/main/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobService.java +++ b/src/main/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobService.java @@ -1,14 +1,11 @@ package org.folio.des.service.bursarlegacy; +import java.util.List; import lombok.RequiredArgsConstructor; -import org.apache.commons.lang3.StringUtils; import org.folio.de.entity.bursarlegacy.JobWithLegacyBursarParameters; -import org.folio.des.domain.dto.JobWithLegacyBursarParametersCollection; -import org.folio.des.repository.CQLService; +import org.folio.des.domain.dto.LegacyBursarFeeFines; import org.folio.des.repository.bursarlegacy.BursarExportLegacyJobRepository; import org.folio.des.service.util.JobMapperUtil; -import org.folio.spring.data.OffsetRequest; -import org.springframework.data.domain.Page; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -18,31 +15,24 @@ public class BursarExportLegacyJobService { private final BursarExportLegacyJobRepository repository; - private final CQLService cqlService; - @Transactional(readOnly = true) - public JobWithLegacyBursarParametersCollection get(Integer offset, Integer limit, String query) { - var result = new JobWithLegacyBursarParametersCollection(); - if (StringUtils.isBlank(query)) { - Page page = repository.findAll( - new OffsetRequest(offset, limit) - ); - result.setJobRecords(page.map(this::entityToDto).getContent()); - result.setTotalRecords((int) page.getTotalElements()); - } else { - result.setJobRecords( - cqlService - .getByCQL(JobWithLegacyBursarParameters.class, query, offset, limit) - .stream() - .map(this::entityToDto) - .toList() - ); - result.setTotalRecords(cqlService.countByCQL(JobWithLegacyBursarParameters.class, query)); - } - return result; + public List getAllLegacyJobs() { + return repository + .findAll() + .stream() + .filter(BursarExportLegacyJobService::hasLegacyBursarParameters) + .map(BursarExportLegacyJobService::entityToDto) + .toList(); + } + + public static boolean hasLegacyBursarParameters(JobWithLegacyBursarParameters job) { + // ensure legacy `bursarFeeFines` is present and actually contains values + LegacyBursarFeeFines legacyObject = job.getExportTypeSpecificParameters().getBursarFeeFines(); + + return (legacyObject != null && legacyObject.getDaysOutstanding() != null); } - private org.folio.des.domain.dto.JobWithLegacyBursarParameters entityToDto(JobWithLegacyBursarParameters entity) { + public static org.folio.des.domain.dto.JobWithLegacyBursarParameters entityToDto(JobWithLegacyBursarParameters entity) { return JobMapperUtil.entityToDto(entity); } } diff --git a/src/main/java/org/folio/des/service/bursarlegacy/BursarMigrationService.java b/src/main/java/org/folio/des/service/bursarlegacy/BursarMigrationService.java index 9bbdde02..c11d24be 100644 --- a/src/main/java/org/folio/des/service/bursarlegacy/BursarMigrationService.java +++ b/src/main/java/org/folio/des/service/bursarlegacy/BursarMigrationService.java @@ -5,7 +5,8 @@ import java.util.List; import java.util.Map; import java.util.UUID; - +import lombok.RequiredArgsConstructor; +import lombok.extern.log4j.Log4j2; import org.folio.des.domain.dto.BursarExportDataToken; import org.folio.des.domain.dto.BursarExportFilter; import org.folio.des.domain.dto.BursarExportFilterAge; @@ -25,223 +26,118 @@ import org.folio.des.domain.dto.BursarExportTokenLengthControl; import org.folio.des.domain.dto.BursarExportTokenUserDataOptional; import org.folio.des.domain.dto.BursarExportTransferCriteria; -import org.folio.des.domain.dto.BursarExportTransferCriteriaConditionsInner; import org.folio.des.domain.dto.BursarExportTransferCriteriaElse; import org.folio.des.domain.dto.ExportTypeSpecificParameters; import org.folio.des.domain.dto.Job; import org.folio.des.domain.dto.JobWithLegacyBursarParameters; -import org.folio.des.domain.dto.JobWithLegacyBursarParametersCollection; import org.folio.des.domain.dto.LegacyBursarFeeFines; import org.folio.des.domain.dto.LegacyBursarFeeFinesTypeMapping; import org.folio.des.domain.dto.LegacyBursarFeeFinesTypeMappings; import org.folio.des.service.JobService; +import org.folio.des.service.util.JobMapperUtil; import org.springframework.stereotype.Service; -import lombok.RequiredArgsConstructor; -import lombok.extern.log4j.Log4j2; - @Service @Log4j2 @RequiredArgsConstructor public class BursarMigrationService { - private static final Integer DEFAULT_LIMIT = 10000; private static final String BURSAR_EXPORT_MIGRATION_HEADER_CONSTANT = "LIB02"; - public boolean isLegacyJob(JobWithLegacyBursarParameters job) { - LegacyBursarFeeFines bursarFeeFines = job - .getExportTypeSpecificParameters() - .getBursarFeeFines(); - - return bursarFeeFines.getDaysOutstanding() != null; - } - - public void recreateLegacyJobs( - BursarExportLegacyJobService bursarExportLegacyJobService, + public void updateLegacyBursarJobs( + BursarExportLegacyJobService legacyJobService, JobService jobService ) { - log.info("searching for legacy jobs"); - JobWithLegacyBursarParametersCollection response = bursarExportLegacyJobService.get( - 0, - DEFAULT_LIMIT, - "status==SCHEDULED" - ); - List jobsToRecreate = new ArrayList<>(); - - int total = response.getTotalRecords(); - jobsToRecreate.addAll(response.getJobRecords()); - while (jobsToRecreate.size() < total) { - response = - bursarExportLegacyJobService.get( - jobsToRecreate.size(), - DEFAULT_LIMIT, - "status==SCHEDULED" - ); - jobsToRecreate.addAll(response.getJobRecords()); - } + List jobs = legacyJobService.getAllLegacyJobs(); + log.info("found {} legacy jobs to update", jobs.size()); - for (JobWithLegacyBursarParameters jobWithLegacyBursarParameters : jobsToRecreate) { - log.info("job to recreate: {}", jobWithLegacyBursarParameters); - if (isLegacyJob(jobWithLegacyBursarParameters)) { - Job newJob = prepareNewJob(); - - newJob - .getExportTypeSpecificParameters() - .setVendorEdiOrdersExportConfig( - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getVendorEdiOrdersExportConfig() - ); - newJob - .getExportTypeSpecificParameters() - .setQuery(jobWithLegacyBursarParameters.getExportTypeSpecificParameters().getQuery()); - newJob - .getExportTypeSpecificParameters() - .seteHoldingsExportConfig( - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .geteHoldingsExportConfig() - ); - newJob - .getExportTypeSpecificParameters() - .setAuthorityControlExportConfig( - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getAuthorityControlExportConfig() - ); + for (JobWithLegacyBursarParameters jobWithLegacyBursarParameters : jobs) { + log.info("updating job: {}", jobWithLegacyBursarParameters); - LegacyBursarFeeFines legacyBursarParams = jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getBursarFeeFines(); - - // filter - BursarExportFilterCondition filterBase = new BursarExportFilterCondition(); - filterBase.setOperation(BursarExportFilterCondition.OperationEnum.AND); - List filterConditions = new ArrayList<>(); - BursarExportFilterAge ageFilter = new BursarExportFilterAge(); - ageFilter.setNumDays(legacyBursarParams.getDaysOutstanding()); - ageFilter.setCondition( - BursarExportFilterAge.ConditionEnum.GREATER_THAN_EQUAL - ); - BursarExportFilterCondition patronGroupListFilter = new BursarExportFilterCondition(); - patronGroupListFilter.setOperation( - BursarExportFilterCondition.OperationEnum.OR - ); - - List patronGroupFilters = new ArrayList<>(); - for (String patronGroupId : legacyBursarParams.getPatronGroups()) { - BursarExportFilterPatronGroup patronGroupFilter = new BursarExportFilterPatronGroup(); - patronGroupFilter.setPatronGroupId(UUID.fromString(patronGroupId)); - - patronGroupFilters.add(patronGroupFilter); - } + Job newJob = convertLegacyJob(jobWithLegacyBursarParameters); - patronGroupListFilter.setCriteria(patronGroupFilters); - - filterConditions.add(ageFilter); - filterConditions.add(patronGroupListFilter); - - filterBase.setCriteria(filterConditions); - - newJob - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setFilter(filterBase); - - // item type token and description token - List typeMappingTokens = mapTypeMappingsToTokens( - legacyBursarParams.getTypeMappings() - ); - BursarExportTokenConditional itemTypeToken = typeMappingTokens.get(0); - BursarExportTokenConditional descriptionToken = typeMappingTokens.get( - 1 - ); - - newJob - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .getData() - .set(3, itemTypeToken); - newJob - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .getData() - .set(9, descriptionToken); - - newJob - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .getTransferInfo() - .getElse() - .setAccount(legacyBursarParams.getTransferAccountId()); - - newJob.setId(jobWithLegacyBursarParameters.getId()); - newJob.setName(jobWithLegacyBursarParameters.getName()); - newJob.setDescription(jobWithLegacyBursarParameters.getDescription()); - newJob.setSource(jobWithLegacyBursarParameters.getSource()); - newJob.setIsSystemSource(jobWithLegacyBursarParameters.getIsSystemSource()); - newJob.setTenant(jobWithLegacyBursarParameters.getTenant()); - newJob.setType(jobWithLegacyBursarParameters.getType()); - newJob.setStatus(jobWithLegacyBursarParameters.getStatus()); - newJob.setFiles(jobWithLegacyBursarParameters.getFiles()); - newJob.setFileNames(jobWithLegacyBursarParameters.getFileNames()); - newJob.setStartTime((jobWithLegacyBursarParameters.getStartTime())); - newJob.setEndTime(jobWithLegacyBursarParameters.getEndTime()); - newJob.setMetadata(jobWithLegacyBursarParameters.getMetadata()); - newJob.setOutputFormat(jobWithLegacyBursarParameters.getOutputFormat()); - newJob.setErrorDetails(jobWithLegacyBursarParameters.getErrorDetails()); - newJob.setIdentifierType(jobWithLegacyBursarParameters.getIdentifierType()); - newJob.setEntityType(jobWithLegacyBursarParameters.getEntityType()); - newJob.setProgress(jobWithLegacyBursarParameters.getProgress()); - - // upsert recreated job - jobService.upsertAndSendToKafka(newJob, true); - } + // upsert recreated job, does not send to kafka despite the name + jobService.upsertAndSendToKafka(newJob, false); } - - log.info("recreated {} legacy jobs", jobsToRecreate.size()); } - public Job prepareNewJob() { - Job job = new Job(); + public static Job convertLegacyJob(JobWithLegacyBursarParameters legacyJob) { + LegacyBursarFeeFines legacyParams = legacyJob + .getExportTypeSpecificParameters() + .getBursarFeeFines(); + ExportTypeSpecificParameters exportTypeSpecificParameters = new ExportTypeSpecificParameters(); BursarExportJob bursarExportJob = new BursarExportJob(); - // filter - BursarExportFilterCondition filterBase = new BursarExportFilterCondition(); - filterBase.setOperation(BursarExportFilterCondition.OperationEnum.AND); + bursarExportJob.setFilter(convertLegacyJobFilters(legacyParams)); + bursarExportJob.setHeader(convertLegacyJobHeader()); + bursarExportJob.setData(convertLegacyJobData(legacyParams)); + bursarExportJob.setFooter(null); + bursarExportJob.setTransferInfo(convertLegacyJobTransfer(legacyParams)); - List filterConditions = new ArrayList<>(); + // new parameters not in legacy jobs + bursarExportJob.setGroupByPatron(false); + bursarExportJob.setGroupByPatronFilter(null); + exportTypeSpecificParameters.setBursarFeeFines(bursarExportJob); + + return JobMapperUtil.legacyBursarToNewDto( + legacyJob, + exportTypeSpecificParameters + ); + } + + private static BursarExportFilterCondition convertLegacyJobFilters( + LegacyBursarFeeFines params + ) { + // age BursarExportFilterAge ageFilter = new BursarExportFilterAge(); - ageFilter.setNumDays(null); + ageFilter.setNumDays(params.getDaysOutstanding()); ageFilter.setCondition( BursarExportFilterAge.ConditionEnum.GREATER_THAN_EQUAL ); - BursarExportFilterCondition patronGroupListFilter = new BursarExportFilterCondition(); patronGroupListFilter.setOperation( BursarExportFilterCondition.OperationEnum.OR ); - List patronGroupFilters = new ArrayList<>(); + // patron groups + List patronGroupFilters = new ArrayList<>(); + for (String patronGroupId : params.getPatronGroups()) { + BursarExportFilterPatronGroup patronGroupFilter = new BursarExportFilterPatronGroup(); + patronGroupFilter.setPatronGroupId(UUID.fromString(patronGroupId)); + patronGroupFilters.add(patronGroupFilter); + } patronGroupListFilter.setCriteria(patronGroupFilters); - filterConditions.add(ageFilter); - filterConditions.add(patronGroupListFilter); + // container + BursarExportFilterCondition filterBase = new BursarExportFilterCondition(); + filterBase.setOperation(BursarExportFilterCondition.OperationEnum.AND); + filterBase.setCriteria(List.of(ageFilter, patronGroupListFilter)); + return filterBase; + } + + // the legacy header consisted just of the string "LIB02" + private static List convertLegacyJobHeader() { + BursarExportTokenConstant constant = new BursarExportTokenConstant(); + constant.setValue(BURSAR_EXPORT_MIGRATION_HEADER_CONSTANT); - filterBase.setCriteria(filterConditions); + BursarExportTokenConstant newline = new BursarExportTokenConstant(); + constant.setValue("\n"); - bursarExportJob.setFilter(filterBase); + return List.of(constant, newline); + } - // header - List header = new ArrayList<>(); - BursarExportTokenConstant constantHeaderToken = new BursarExportTokenConstant(); - constantHeaderToken.setValue(BURSAR_EXPORT_MIGRATION_HEADER_CONSTANT); - header.add(constantHeaderToken); - bursarExportJob.setHeader(header); + private static List convertLegacyJobData( + LegacyBursarFeeFines params + ) { + List typeMappingTokens = mapTypeMappingsToTokens( + params.getTypeMappings() + ); + // special if-else tokens for item type and description + BursarExportTokenConditional itemTypeToken = typeMappingTokens.get(0); + BursarExportTokenConditional descriptionToken = typeMappingTokens.get(1); - //data //user's external id token BursarExportTokenUserDataOptional userIDToken = new BursarExportTokenUserDataOptional(); userIDToken.setValue( @@ -309,69 +205,35 @@ public Job prepareNewJob() { BursarExportTokenConstant termToken = new BursarExportTokenConstant(); termToken.setValue(" "); - // item type token and description token - BursarExportTokenConditional itemTypeToken = new BursarExportTokenConditional(); - BursarExportTokenConditional descriptionToken = new BursarExportTokenConditional(); - - List itemTypeConditions = new ArrayList<>(); - List descriptionConditions = new ArrayList<>(); - - itemTypeToken.setConditions(itemTypeConditions); - descriptionToken.setConditions(descriptionConditions); - - // default for itemType token - BursarExportTokenConstant emptyDataToken = new BursarExportTokenConstant(); - emptyDataToken.setValue(" "); - itemTypeToken.setElse(emptyDataToken); - - // default for description token - BursarExportTokenFeeMetadata feeFineTypeToken = new BursarExportTokenFeeMetadata(); - feeFineTypeToken.setValue(BursarExportTokenFeeMetadata.ValueEnum.NAME); - descriptionToken.setElse(feeFineTypeToken); - - List dataTokens = new ArrayList<>( - Arrays.asList( - userIDToken, - userIdPadding, - feeAmountToken, - itemTypeToken, - monthToken, - dayToken, - yearToken, - sfsToken, - termToken, - descriptionToken - ) + return List.of( + userIDToken, + userIdPadding, + feeAmountToken, + itemTypeToken, + monthToken, + dayToken, + yearToken, + sfsToken, + termToken, + descriptionToken ); + } - bursarExportJob.setData(dataTokens); - - // footer - bursarExportJob.setFooter(null); + private static BursarExportTransferCriteria convertLegacyJobTransfer( + LegacyBursarFeeFines legacyParams + ) { + // legacy exports would transfer all to one account, so we don't have any conditions and use + // the account as the default (else) + BursarExportTransferCriteriaElse transferElse = new BursarExportTransferCriteriaElse(); + transferElse.setAccount(legacyParams.getTransferAccountId()); - // transfer information BursarExportTransferCriteria transferCriteria = new BursarExportTransferCriteria(); - - List transferConditions = new ArrayList<>(); - - BursarExportTransferCriteriaElse transferInfo = new BursarExportTransferCriteriaElse(); - - transferCriteria.setConditions(transferConditions); - transferCriteria.setElse(transferInfo); - - bursarExportJob.setTransferInfo(transferCriteria); - - // other bursar export parameters - bursarExportJob.setGroupByPatron(false); - bursarExportJob.setGroupByPatronFilter(null); - - exportTypeSpecificParameters.setBursarFeeFines(bursarExportJob); - job.setExportTypeSpecificParameters(exportTypeSpecificParameters); - - return job; + transferCriteria.setConditions(List.of()); + transferCriteria.setElse(transferElse); + return transferCriteria; } - public List mapTypeMappingsToTokens( + public static List mapTypeMappingsToTokens( LegacyBursarFeeFinesTypeMappings typeMappings ) { // item type token and description token @@ -385,9 +247,14 @@ public List mapTypeMappingsToTokens( descriptionToken.setConditions(descriptionConditions); if (typeMappings != null) { - for (Map.Entry> entry : typeMappings.entrySet()) { + for (Map.Entry entry : typeMappings.entrySet()) { String ownerID = entry.getKey(); - List typeMappingList = entry.getValue(); + + // unfortunately, LegacyBursarFeeFinesTypeMappings is generated by openapi, which gives it only Map + // this casts it to the correct type + @SuppressWarnings("unchecked") + List typeMappingList = (List) entry.getValue(); + for (LegacyBursarFeeFinesTypeMapping typeMapping : typeMappingList) { BursarExportTokenConditionalConditionsInner itemTypeConditionalConditionsInner = new BursarExportTokenConditionalConditionsInner(); BursarExportTokenConditionalConditionsInner descriptionTokenConditionalConditionsInner = new BursarExportTokenConditionalConditionsInner(); @@ -433,12 +300,10 @@ public List mapTypeMappingsToTokens( } } - // default for itemType token BursarExportTokenConstant emptyDataToken = new BursarExportTokenConstant(); emptyDataToken.setValue(" "); itemTypeToken.setElse(emptyDataToken); - // default for description token BursarExportTokenFeeMetadata feeFineTypeToken = new BursarExportTokenFeeMetadata(); feeFineTypeToken.setValue(BursarExportTokenFeeMetadata.ValueEnum.NAME); descriptionToken.setElse(feeFineTypeToken); diff --git a/src/main/java/org/folio/des/service/util/JobMapperUtil.java b/src/main/java/org/folio/des/service/util/JobMapperUtil.java index f694a711..5540a7c7 100644 --- a/src/main/java/org/folio/des/service/util/JobMapperUtil.java +++ b/src/main/java/org/folio/des/service/util/JobMapperUtil.java @@ -4,13 +4,13 @@ import org.folio.de.entity.Job; import org.folio.de.entity.bursarlegacy.JobWithLegacyBursarParameters; import org.folio.des.domain.dto.ExportType; +import org.folio.des.domain.dto.ExportTypeSpecificParameters; import org.folio.des.domain.dto.Metadata; -public class JobMapperUtil { +import lombok.experimental.UtilityClass; - private JobMapperUtil() { - throw new IllegalStateException("Utility class"); - } +@UtilityClass +public class JobMapperUtil { public static org.folio.des.domain.dto.Job entityToDto(Job jobEntity) { var result = new org.folio.des.domain.dto.Job(); @@ -95,4 +95,37 @@ public static org.folio.des.domain.dto.JobWithLegacyBursarParameters entityToDto return result; } + + public static org.folio.des.domain.dto.Job legacyBursarToNewDto( + org.folio.des.domain.dto.JobWithLegacyBursarParameters jobWithLegacyBursarParametersEntity, + ExportTypeSpecificParameters newExportTypeSpecificParameters + ) { + var result = new org.folio.des.domain.dto.Job(); + + // this is the only different field between the two + result.setExportTypeSpecificParameters(newExportTypeSpecificParameters); + + result.setId(jobWithLegacyBursarParametersEntity.getId()); + result.setName(jobWithLegacyBursarParametersEntity.getName()); + result.setDescription(jobWithLegacyBursarParametersEntity.getDescription()); + result.setSource(jobWithLegacyBursarParametersEntity.getSource()); + result.setIsSystemSource(jobWithLegacyBursarParametersEntity.getIsSystemSource()); + result.setTenant(jobWithLegacyBursarParametersEntity.getTenant()); + result.setType(jobWithLegacyBursarParametersEntity.getType()); + result.setStatus(jobWithLegacyBursarParametersEntity.getStatus()); + result.setFiles(jobWithLegacyBursarParametersEntity.getFiles()); + result.setFileNames(jobWithLegacyBursarParametersEntity.getFileNames()); + result.setStartTime(jobWithLegacyBursarParametersEntity.getStartTime()); + result.setEndTime(jobWithLegacyBursarParametersEntity.getEndTime()); + result.setIdentifierType(jobWithLegacyBursarParametersEntity.getIdentifierType()); + result.setEntityType(jobWithLegacyBursarParametersEntity.getEntityType()); + result.setProgress(jobWithLegacyBursarParametersEntity.getProgress()); + + result.setMetadata(jobWithLegacyBursarParametersEntity.getMetadata()); + + result.setOutputFormat(jobWithLegacyBursarParametersEntity.getOutputFormat()); + result.setErrorDetails(jobWithLegacyBursarParametersEntity.getErrorDetails()); + + return result; + } } diff --git a/src/main/resources/swagger.api/jobs.yaml b/src/main/resources/swagger.api/jobs.yaml index 1f711c8a..280b70b9 100644 --- a/src/main/resources/swagger.api/jobs.yaml +++ b/src/main/resources/swagger.api/jobs.yaml @@ -226,8 +226,6 @@ components: $ref: '../../../../folio-export-common/schemas/job.json#/Job' jobWithLegacyBursarParameters: $ref: '../../../../folio-export-common/schemas/jobWithLegacyBursarParameters.json#/JobWithLegacyBursarParameters' - jobWithLegacyBursarParametersCollection: - $ref: '../../../../folio-export-common/schemas/jobWithLegacyBursarParametersCollection.json#/JobWithLegacyBursarParametersCollection' jobCollection: $ref: '../../../../folio-export-common/schemas/jobCollection.json#/JobCollection' userdataCollection: diff --git a/src/test/java/org/folio/des/service/FolioTenantServiceTest.java b/src/test/java/org/folio/des/service/FolioTenantServiceTest.java index dae4d967..d236b45a 100644 --- a/src/test/java/org/folio/des/service/FolioTenantServiceTest.java +++ b/src/test/java/org/folio/des/service/FolioTenantServiceTest.java @@ -30,12 +30,16 @@ class FolioTenantServiceTest { @Mock KafkaService kafka; + @Mock BulkEditConfigService bulkEditConfigService; + @Mock EdifactScheduledJobInitializer edifactScheduledJobInitializer; + @Mock FolioExecutionContext folioExecutionContext; + @Mock ScheduledJobsRemover scheduledJobsRemover; @@ -44,6 +48,7 @@ class FolioTenantServiceTest { @Mock OldJobDeleteScheduler oldJobDeleteScheduler; + @Mock PrepareSystemUserService prepareSystemUserService; @@ -56,19 +61,27 @@ void shouldDoProcessAfterTenantUpdating() { doNothing().when(prepareSystemUserService).setupSystemUser(); doNothing().when(bulkEditConfigService).checkBulkEditConfiguration(); - doNothing().when(edifactScheduledJobInitializer).initAllScheduledJob(tenantAttributes); + doNothing() + .when(edifactScheduledJobInitializer) + .initAllScheduledJob(tenantAttributes); doNothing().when(kafka).createKafkaTopics(); doNothing().when(kafka).restartEventListeners(); - doNothing().when(bursarScheduledJobInitializer).initAllScheduledJob(tenantAttributes); + doNothing() + .when(bursarScheduledJobInitializer) + .initAllScheduledJob(tenantAttributes); doNothing().when(oldJobDeleteScheduler).scheduleOldJobDeletion(any()); - doNothing().when(bursarMigrationService).recreateLegacyJobs(any(), any()); + doNothing() + .when(bursarMigrationService) + .updateLegacyBursarJobs(any(), any()); folioTenantService.afterTenantUpdate(tenantAttributes); verify(prepareSystemUserService).setupSystemUser(); verify(bulkEditConfigService, times(1)).checkBulkEditConfiguration(); - verify(edifactScheduledJobInitializer, times(1)).initAllScheduledJob(tenantAttributes); - verify(bursarScheduledJobInitializer, times(1)).initAllScheduledJob(tenantAttributes); + verify(edifactScheduledJobInitializer, times(1)) + .initAllScheduledJob(tenantAttributes); + verify(bursarScheduledJobInitializer, times(1)) + .initAllScheduledJob(tenantAttributes); verify(oldJobDeleteScheduler, times(1)).scheduleOldJobDeletion(any()); verify(kafka, times(1)).createKafkaTopics(); verify(kafka, times(1)).restartEventListeners(); diff --git a/src/test/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobServiceTest.java b/src/test/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobServiceTest.java index f9746272..2ec313d4 100644 --- a/src/test/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobServiceTest.java +++ b/src/test/java/org/folio/des/service/bursarlegacy/BursarExportLegacyJobServiceTest.java @@ -3,22 +3,19 @@ import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.when; -import java.util.ArrayList; import java.util.List; import java.util.UUID; - import org.folio.de.entity.bursarlegacy.JobWithLegacyBursarParameters; -import org.folio.des.domain.dto.ExportType; -import org.folio.des.domain.dto.JobWithLegacyBursarParametersCollection; -import org.folio.des.repository.CQLService; +import org.folio.des.domain.dto.ExportTypeSpecificParametersWithLegacyBursar; +import org.folio.des.domain.dto.LegacyBursarFeeFines; import org.folio.des.repository.bursarlegacy.BursarExportLegacyJobRepository; -import org.folio.spring.data.OffsetRequest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.PageImpl; @ExtendWith(MockitoExtension.class) class BursarExportLegacyJobServiceTest { @@ -26,38 +23,75 @@ class BursarExportLegacyJobServiceTest { @Mock private BursarExportLegacyJobRepository repository; - @Mock - private CQLService cqlService; + static JobWithLegacyBursarParameters nonBursar1, nonBursar2, nonBursar3, isBursar; + + static { + nonBursar1 = new JobWithLegacyBursarParameters(); + nonBursar1.setId(UUID.fromString("998a84d7-9538-5652-8a40-d26bbb8507bc")); + nonBursar1.setExportTypeSpecificParameters( + new ExportTypeSpecificParametersWithLegacyBursar() + ); + + nonBursar2 = new JobWithLegacyBursarParameters(); + nonBursar2.setId(UUID.fromString("dfad2baa-c970-5933-b86a-68781c5c4f2c")); + nonBursar2.setExportTypeSpecificParameters( + new ExportTypeSpecificParametersWithLegacyBursar().bursarFeeFines(null) + ); + + nonBursar3 = new JobWithLegacyBursarParameters(); + nonBursar3.setId(UUID.fromString("aae30cd1-4549-5572-bb14-4c04c1b28366")); + nonBursar3.setExportTypeSpecificParameters( + new ExportTypeSpecificParametersWithLegacyBursar() + .bursarFeeFines(new LegacyBursarFeeFines()) + ); + + isBursar = new JobWithLegacyBursarParameters(); + isBursar.setId(UUID.fromString("5a0e5c0b-a216-552c-8a28-6b25ada9ae84")); + isBursar.setExportTypeSpecificParameters( + new ExportTypeSpecificParametersWithLegacyBursar() + .bursarFeeFines(new LegacyBursarFeeFines().daysOutstanding(1)) + ); + } @Test void testGetBlankQuery() { - List legacyJobs = new ArrayList<>(); - JobWithLegacyBursarParameters legacyJob1 = new JobWithLegacyBursarParameters(); - legacyJob1.setId(UUID.fromString("0000-00-00-00-000000")); - legacyJobs.add(legacyJob1); - JobWithLegacyBursarParameters legacyJob2 = new JobWithLegacyBursarParameters(); - legacyJob2.setId(UUID.fromString("0000-00-00-00-000001")); - legacyJob2.setType(ExportType.EDIFACT_ORDERS_EXPORT); - legacyJobs.add(legacyJob2); - - Page page = new PageImpl( - legacyJobs, - new OffsetRequest(0, 1), - 1L - ); - - when(repository.findAll(new OffsetRequest(0, 2))).thenReturn(page); + List items = List.of( + nonBursar1, + nonBursar2, + nonBursar3, + isBursar + ); + + when(repository.findAll()).thenReturn(items); + BursarExportLegacyJobService service = new BursarExportLegacyJobService( - repository, - cqlService + repository ); - JobWithLegacyBursarParametersCollection legacyJobCollection = service.get(0, 2, ""); + List legacyJobCollection = service.getAllLegacyJobs(); + + assertEquals(1, legacyJobCollection.size()); + assertEquals(isBursar.getId(), legacyJobCollection.get(0).getId()); + } + + private static List hasLegacyParametersArguments() { + return List.of( + Arguments.of(isBursar, true), + Arguments.of(nonBursar1, false), + Arguments.of(nonBursar2, false), + Arguments.of(nonBursar3, false) + ); + } - assertEquals(2, legacyJobCollection.getJobRecords().size()); + @ParameterizedTest + @MethodSource("hasLegacyParametersArguments") + void testHasLegacyBursarParameters( + JobWithLegacyBursarParameters job, + boolean expected + ) { assertEquals( - UUID.fromString("0000-00-00-00-000000"), - legacyJobCollection.getJobRecords().get(0).getId() + expected, + BursarExportLegacyJobService.hasLegacyBursarParameters(job) ); } } diff --git a/src/test/java/org/folio/des/service/bursarlegacy/BursarMigrationServiceTest.java b/src/test/java/org/folio/des/service/bursarlegacy/BursarMigrationServiceTest.java index 0f7278ad..5944e1cb 100644 --- a/src/test/java/org/folio/des/service/bursarlegacy/BursarMigrationServiceTest.java +++ b/src/test/java/org/folio/des/service/bursarlegacy/BursarMigrationServiceTest.java @@ -1,14 +1,12 @@ package org.folio.des.service.bursarlegacy; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; -import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.UUID; import org.folio.des.domain.dto.BursarExportFilter; @@ -22,17 +20,14 @@ import org.folio.des.domain.dto.ExportTypeSpecificParametersWithLegacyBursar; import org.folio.des.domain.dto.Job; import org.folio.des.domain.dto.JobWithLegacyBursarParameters; -import org.folio.des.domain.dto.JobWithLegacyBursarParametersCollection; import org.folio.des.domain.dto.LegacyBursarFeeFines; import org.folio.des.domain.dto.LegacyBursarFeeFinesTypeMapping; import org.folio.des.domain.dto.LegacyBursarFeeFinesTypeMappings; import org.folio.des.service.JobService; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.InjectMocks; import org.mockito.Mock; -import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; @ExtendWith(MockitoExtension.class) @@ -48,177 +43,111 @@ class BursarMigrationServiceTest { private JobService jobService; @Test - void testIsLegacyJob() { - JobWithLegacyBursarParameters job = mockJob(); + void testNoLegacyJobs() { + when(bursarExportLegacyJobService.getAllLegacyJobs()).thenReturn(List.of()); - JobWithLegacyBursarParameters jobWithLegacyBursarParameters = mockJob(); - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setDaysOutstanding(1); - - Assertions.assertAll( - () -> assertFalse(bursarMigrationService.isLegacyJob(job)), - () -> assertTrue(bursarMigrationService.isLegacyJob(jobWithLegacyBursarParameters)) - ); - } - - @Test - void testRecreateNoLegacyJob() { - List jobsWithLegacyBursarParameters = new ArrayList<>(); - JobWithLegacyBursarParameters job = mockJob(); - job.setId(UUID.fromString("0000-00-00-00-000000")); - jobsWithLegacyBursarParameters.add(job); - - Mockito - .when(bursarExportLegacyJobService.get(0, 10000, "status==SCHEDULED")) - .thenReturn(mockLegacyCollectionOneItem(jobsWithLegacyBursarParameters)); - - bursarMigrationService.recreateLegacyJobs( - bursarExportLegacyJobService, - jobService - ); - - Mockito - .verify(jobService, times(0)) - .upsertAndSendToKafka(any(Job.class), eq(true)); - } - - @Test - void testRecreateOneLegacyJob() { - List jobsWithLegacyBursarParameters = new ArrayList<>(); - JobWithLegacyBursarParameters jobWithLegacyBursarParameters = mockJob(); - jobWithLegacyBursarParameters.setId(UUID.fromString("0000-00-00-00-000000")); - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setDaysOutstanding(1); - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setPatronGroups(Arrays.asList("0000-00-00-00-000001")); - jobsWithLegacyBursarParameters.add(jobWithLegacyBursarParameters); - - Job job = bursarMigrationService.prepareNewJob(); - job.setId(UUID.fromString("0000-00-00-00-000000")); - BursarExportFilterCondition filterBase = new BursarExportFilterCondition(); - filterBase.setOperation(BursarExportFilterCondition.OperationEnum.AND); - List filterConditions = new ArrayList<>(); - BursarExportFilterAge ageFilter = new BursarExportFilterAge(); - ageFilter.setNumDays(1); - ageFilter.setCondition( - BursarExportFilterAge.ConditionEnum.GREATER_THAN_EQUAL - ); - BursarExportFilterCondition patronGroupListFilter = new BursarExportFilterCondition(); - patronGroupListFilter.setOperation( - BursarExportFilterCondition.OperationEnum.OR - ); - List patronGroupFilters = new ArrayList<>(); - BursarExportFilterPatronGroup patronGroupFilter = new BursarExportFilterPatronGroup(); - patronGroupFilter.setPatronGroupId(UUID.fromString("0000-00-00-00-000001")); - patronGroupFilters.add(patronGroupFilter); - patronGroupListFilter.setCriteria(patronGroupFilters); - - filterConditions.add(ageFilter); - filterConditions.add(patronGroupListFilter); - - filterBase.setCriteria(filterConditions); - - job - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setFilter(filterBase); - - Mockito - .when(bursarExportLegacyJobService.get(0, 10000, "status==SCHEDULED")) - .thenReturn(mockLegacyCollectionOneItem(jobsWithLegacyBursarParameters)); - - bursarMigrationService.recreateLegacyJobs( + bursarMigrationService.updateLegacyBursarJobs( bursarExportLegacyJobService, jobService ); - Mockito.verify(jobService, times(1)).upsertAndSendToKafka(job, true); + verifyNoInteractions(jobService); } @Test - void testRecreateMultipleLegacyJobs() { - // set up legacy job - List jobsWithLegacyBursarParameters = new ArrayList<>(); - JobWithLegacyBursarParameters jobWithLegacyBursarParameters = mockJob(); - jobWithLegacyBursarParameters.setId(UUID.fromString("0000-00-00-00-000000")); - jobWithLegacyBursarParameters - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setDaysOutstanding(1); - - jobsWithLegacyBursarParameters.add(jobWithLegacyBursarParameters); - - Job job = bursarMigrationService.prepareNewJob(); - job.setId(UUID.fromString("0000-00-00-00-000000")); - BursarExportFilterCondition filterBase = new BursarExportFilterCondition(); - filterBase.setOperation(BursarExportFilterCondition.OperationEnum.AND); - List filterConditions = new ArrayList<>(); - BursarExportFilterAge ageFilter = new BursarExportFilterAge(); // outstandingDays => ageFilter - ageFilter.setNumDays(1); - ageFilter.setCondition( - BursarExportFilterAge.ConditionEnum.GREATER_THAN_EQUAL - ); - BursarExportFilterCondition patronGroupListFilter = new BursarExportFilterCondition(); // patronGroups => patronGroupListFilter - patronGroupListFilter.setOperation( - BursarExportFilterCondition.OperationEnum.OR - ); - List patronGroupFilters = new ArrayList<>(); - patronGroupListFilter.setCriteria(patronGroupFilters); - - filterConditions.add(ageFilter); - filterConditions.add(patronGroupListFilter); - - filterBase.setCriteria(filterConditions); - - job - .getExportTypeSpecificParameters() - .getBursarFeeFines() - .setFilter(filterBase); - - JobWithLegacyBursarParametersCollection legacyCollection = mockLegacyCollectionOneItem( - jobsWithLegacyBursarParameters + void testLegacyJobConversion() { + JobWithLegacyBursarParameters legacyJob = new JobWithLegacyBursarParameters(); + legacyJob.setId(UUID.fromString("928f4cc0-0f44-5d5f-aa73-3b7bc532961e")); + legacyJob.setExportTypeSpecificParameters( + new ExportTypeSpecificParametersWithLegacyBursar() + .bursarFeeFines( + new LegacyBursarFeeFines() + .daysOutstanding(1) + .addPatronGroupsItem("ed98f8c2-09b0-5d46-b610-ba955d0bf303") + .addPatronGroupsItem("d82b6807-4ab3-5412-8428-1b49ac20e0c2") + ) + ); + + // check converted job makes sense + Job job = BursarMigrationService.convertLegacyJob(legacyJob); + assertEquals( + "928f4cc0-0f44-5d5f-aa73-3b7bc532961e", + job.getId().toString() ); - legacyCollection.setTotalRecords(2); - Mockito - .when(bursarExportLegacyJobService.get(0, 10000, "status==SCHEDULED")) - .thenReturn(legacyCollection); - Mockito - .when(bursarExportLegacyJobService.get(1, 10000, "status==SCHEDULED")) - .thenReturn(legacyCollection); + // verify filters + List filters = + ( + (BursarExportFilterCondition) job + .getExportTypeSpecificParameters() + .getBursarFeeFines() + .getFilter() + ).getCriteria(); - bursarMigrationService.recreateLegacyJobs( + // age filter + assertEquals( + 1, + filters + .stream() + .filter(i -> i instanceof BursarExportFilterAge) + .map(i -> (BursarExportFilterAge) i) + .findFirst() + .get() + .getNumDays() + ); + + // patron filters are nested inside an OR + List patronFilters = filters + .stream() + .filter(i -> i instanceof BursarExportTokenConditional) + .map(i -> (BursarExportTokenConditional) i) + .findFirst() + .get() + .getConditions() + .stream() + .map(i -> (BursarExportFilterPatronGroup) i.getCondition()) + .toList(); + assertEquals(2, patronFilters.size()); + assertTrue( + patronFilters + .stream() + .map(BursarExportFilterPatronGroup::getPatronGroupId) + .anyMatch(i -> i.equals(UUID.fromString("ed98f8c2-09b0-5d46-b610-ba955d0bf303"))) + ); + assertTrue( + patronFilters + .stream() + .map(BursarExportFilterPatronGroup::getPatronGroupId) + .anyMatch(i -> i.equals(UUID.fromString("d82b6807-4ab3-5412-8428-1b49ac20e0c2"))) + ); + + // ensure that calling .updateLegacyBursarJobs correctly updates the job above + when(bursarExportLegacyJobService.getAllLegacyJobs()) + .thenReturn(List.of(legacyJob)); + + bursarMigrationService.updateLegacyBursarJobs( bursarExportLegacyJobService, jobService ); - Mockito - .verify(bursarExportLegacyJobService, times(2)) - .get(any(), eq(10000), eq("status==SCHEDULED")); - Mockito.verify(jobService, times(2)).upsertAndSendToKafka(job, true); + verify(bursarExportLegacyJobService, times(1)).getAllLegacyJobs(); + verify(jobService, times(1)).upsertAndSendToKafka(job, false); } @Test void testMapTypeMappingsToTokens() { LegacyBursarFeeFinesTypeMappings typeMappings = new LegacyBursarFeeFinesTypeMappings(); - List typeMappingList = new ArrayList<>(); LegacyBursarFeeFinesTypeMapping typeMapping = new LegacyBursarFeeFinesTypeMapping(); - typeMapping.setFeefineTypeId(UUID.fromString("0000-00-00-00-000001")); + typeMapping.setFeefineTypeId(UUID.fromString("c4ff3edb-2cc4-523c-a90d-9a2fc8b02a00")); typeMapping.setItemType("test_item_type"); typeMapping.setItemDescription("test_item_description"); - typeMappingList.add(typeMapping); - typeMappings.put("0000-00-00-00-000001", typeMappingList); - - List tokens = bursarMigrationService.mapTypeMappingsToTokens( - typeMappings + typeMappings.put( + "81a5bb0a-e3a1-57b5-b8a8-9324989a7585", + List.of(typeMapping) ); + List tokens = BursarMigrationService.mapTypeMappingsToTokens(typeMappings); + // Type checking for item token assertEquals( BursarExportFilterCondition.class, @@ -239,7 +168,7 @@ void testMapTypeMappingsToTokens() { .getCriteria() .get(0); assertEquals( - UUID.fromString("0000-00-00-00-000001"), + UUID.fromString("c4ff3edb-2cc4-523c-a90d-9a2fc8b02a00"), feeType.getFeeFineTypeId() ); assertEquals( @@ -250,7 +179,7 @@ void testMapTypeMappingsToTokens() { .getCriteria() .get(1); assertEquals( - UUID.fromString("0000-00-00-00-000001"), + UUID.fromString("81a5bb0a-e3a1-57b5-b8a8-9324989a7585"), feeFineOwner.getFeeFineOwner() ); assertEquals( @@ -288,27 +217,4 @@ void testMapTypeMappingsToTokens() { ).getValue() ); } - - private JobWithLegacyBursarParameters mockJob() { - JobWithLegacyBursarParameters job = new JobWithLegacyBursarParameters(); - ExportTypeSpecificParametersWithLegacyBursar nonLegacyExportTypeSpecificParameters = new ExportTypeSpecificParametersWithLegacyBursar(); - LegacyBursarFeeFines nonLegacyBursarFeeFines = new LegacyBursarFeeFines(); - nonLegacyExportTypeSpecificParameters.setBursarFeeFines(nonLegacyBursarFeeFines); - job.setExportTypeSpecificParameters( - nonLegacyExportTypeSpecificParameters - ); - - return job; - } - - private JobWithLegacyBursarParametersCollection mockLegacyCollectionOneItem( - List jobsWithLegacyBursarParameters - ) { - JobWithLegacyBursarParametersCollection legacyCollection = new JobWithLegacyBursarParametersCollection(); - - legacyCollection.setJobRecords(jobsWithLegacyBursarParameters); - legacyCollection.setTotalRecords(1); - - return legacyCollection; - } }