MODBULKOPS-429 - Commit changes - MARC Bib records with administrativ…

…e data (#356)

src/main/java/org/folio/bulkops/controller/BulkOperationController.java

@@ -3,6 +3,7 @@
 import static org.folio.bulkops.domain.dto.EntityType.INSTANCE;
 import static org.folio.bulkops.domain.dto.EntityType.INSTANCE_MARC;
 import static org.folio.bulkops.domain.dto.FileContentType.COMMITTED_RECORDS_FILE;
+import static org.folio.bulkops.domain.dto.FileContentType.COMMITTED_RECORDS_MARC_FILE;
 import static org.folio.bulkops.domain.dto.FileContentType.COMMITTING_CHANGES_ERROR_FILE;
 import static org.folio.bulkops.domain.dto.FileContentType.MATCHED_RECORDS_FILE;
 import static org.folio.bulkops.domain.dto.FileContentType.PROPOSED_CHANGES_FILE;
@@ -164,9 +165,9 @@ public ResponseEntity<Resource> downloadFileByOperationId(
     } else if (fileContentType == PROPOSED_CHANGES_MARC_FILE) {
       path = bulkOperation.getLinkToModifiedRecordsMarcFile();
     } else if (fileContentType == COMMITTED_RECORDS_FILE) {
-      path = INSTANCE_MARC.equals(bulkOperation.getEntityType()) ?
-        bulkOperation.getLinkToCommittedRecordsMarcFile() :
-        bulkOperation.getLinkToCommittedRecordsCsvFile();
+      path = bulkOperation.getLinkToCommittedRecordsCsvFile();
+    } else if (fileContentType == COMMITTED_RECORDS_MARC_FILE) {
+      path = bulkOperation.getLinkToCommittedRecordsMarcFile();
     } else if (fileContentType == COMMITTING_CHANGES_ERROR_FILE) {
       path = bulkOperation.getLinkToCommittedRecordsErrorsCsvFile();
     } else {

src/main/java/org/folio/bulkops/processor/folio/FolioInstanceUpdateProcessor.java

@@ -3,12 +3,14 @@
 import static java.lang.Boolean.TRUE;
 import static java.lang.Boolean.parseBoolean;
 import static java.lang.String.format;
+import static org.folio.bulkops.domain.dto.EntityType.INSTANCE_MARC;
 import static org.folio.bulkops.domain.dto.UpdateOptionType.SUPPRESS_FROM_DISCOVERY;
 import static org.folio.bulkops.util.Constants.APPLY_TO_HOLDINGS;
 import static org.folio.bulkops.util.Constants.APPLY_TO_ITEMS;
 import static org.folio.bulkops.util.Constants.GET_HOLDINGS_BY_INSTANCE_ID_QUERY;
 import static org.folio.bulkops.util.Constants.GET_ITEMS_BY_HOLDING_ID_QUERY;
 import static org.folio.bulkops.util.Constants.MARC;
+import static org.folio.bulkops.util.Constants.MSG_NO_ADMINISTRATIVE_CHANGE_REQUIRED;
 import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED;
 import static org.folio.bulkops.util.FolioExecutionContextUtil.prepareContextForTenant;
 import static org.folio.bulkops.util.RuleUtils.fetchParameters;
@@ -83,7 +85,7 @@ public void updateAssociatedRecords(ExtendedInstance extendedInstance, BulkOpera
       .filter(rule -> applyRuleToAssociatedRecords(extendedInstance, rule, operation))
       .isPresent();
     if (notChanged) {
-      var errorMessage = buildErrorMessage(recordsUpdated, instance.getDiscoverySuppress());
+      var errorMessage = buildErrorMessage(recordsUpdated, instance.getDiscoverySuppress(), operation.getEntityType());
       errorService.saveError(operation.getId(), instance.getIdentifier(operation.getIdentifierType()), errorMessage, ErrorType.WARNING);
     }
   }
@@ -168,11 +170,14 @@ private boolean suppressItemsIfRequired(List<HoldingsRecord> holdingsRecords, bo
     return true;
   }
 
-  private String buildErrorMessage(boolean recordsUpdated, boolean newValue) {
+  private String buildErrorMessage(boolean recordsUpdated, boolean newValue, EntityType entityType) {
     var affectedState = TRUE.equals(newValue) ? "unsuppressed" : "suppressed";
+    var message = INSTANCE_MARC.equals(entityType) ?
+      MSG_NO_ADMINISTRATIVE_CHANGE_REQUIRED :
+      MSG_NO_CHANGE_REQUIRED;
     return recordsUpdated ?
       format(ERROR_MESSAGE_TEMPLATE, affectedState) :
-      MSG_NO_CHANGE_REQUIRED;
+      message;
   }
 
   private List<HoldingsRecord> getHoldingsSourceFolioByInstanceId(String instanceId) {

src/main/java/org/folio/bulkops/repository/BulkOperationExecutionRepository.java

@@ -1,5 +1,6 @@
 package org.folio.bulkops.repository;
 
+import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -10,4 +11,5 @@
 @Repository
 public interface BulkOperationExecutionRepository extends JpaRepository<BulkOperationExecution, UUID> {
   Optional<BulkOperationExecution> findByBulkOperationId(UUID uuid);
+  List<BulkOperationExecution> findAllByBulkOperationId(UUID uuid);
 }

src/main/java/org/folio/bulkops/service/BulkOperationService.java

@@ -1,6 +1,7 @@
 package org.folio.bulkops.service;
 
 import static java.lang.String.format;
+import static java.util.Objects.isNull;
 import static java.util.Objects.nonNull;
 import static org.apache.commons.lang3.StringUtils.EMPTY;
 import static org.apache.commons.lang3.StringUtils.LF;
@@ -152,7 +153,7 @@ public class BulkOperationService {
   private static final String PREVIEW_CSV_PATH_TEMPLATE = "%s/%s-Updates-Preview-CSV-%s.csv";
   private static final String PREVIEW_MARC_PATH_TEMPLATE = "%s/%s-Updates-Preview-MARC-%s.mrc";
   private static final String CHANGED_JSON_PATH_TEMPLATE = "%s/json/%s-Changed-Records-%s.json";
-  private static final String CHANGED_CSV_PATH_TEMPLATE = "%s/%s-Changed-Records-%s.csv";
+  private static final String CHANGED_CSV_PATH_TEMPLATE = "%s/%s-Changed-Records-CSV-%s.csv";
 
   private final ExecutorService executor = Executors.newCachedThreadPool();
 
@@ -379,16 +380,17 @@ public void commit(BulkOperation operation) {
     var operationId = operation.getId();
     operation.setCommittedNumOfRecords(0);
     operation.setStatus(OperationStatusType.APPLY_CHANGES);
-    operation.setTotalNumOfRecords(operation.getMatchedNumOfRecords());
 
-    operation = bulkOperationRepository.save(operation);
+    var totalNumOfRecords = operation.getMatchedNumOfRecords();
 
-    if (operation.getEntityType() == INSTANCE_MARC) {
+    if (INSTANCE_MARC.equals(operation.getEntityType())) {
       marcUpdateService.saveErrorsForFolioInstances(operation);
-      marcUpdateService.commitForInstanceMarc(operation);
-      return;
+      totalNumOfRecords *= getProgressMultiplier(operation);
     }
 
+    operation.setTotalNumOfRecords(totalNumOfRecords);
+    operation = bulkOperationRepository.save(operation);
+
     if (StringUtils.isNotEmpty(operation.getLinkToModifiedRecordsJsonFile())) {
       var entityClass = resolveEntityClass(operation.getEntityType());
       var extendedClass = resolveExtendedEntityClass(operation.getEntityType());
@@ -462,8 +464,6 @@ public void commit(BulkOperation operation) {
         }
 
         execution.setProcessedRecords(processedNumOfRecords);
-        operation.setProcessedNumOfRecords(operation.getCommittedNumOfRecords());
-        operation.setEndTime(LocalDateTime.now());
         if (operation.getCommittedNumOfRecords() > 0) {
           operation.setLinkToCommittedRecordsCsvFile(resultCsvFileName);
           operation.setLinkToCommittedRecordsJsonFile(resultJsonFileName);
@@ -479,15 +479,21 @@ public void commit(BulkOperation operation) {
       executionRepository.save(execution);
     }
 
-    var linkToCommittingErrorsFile = errorService.uploadErrorsToStorage(operationId);
-    operation.setLinkToCommittedRecordsErrorsCsvFile(linkToCommittingErrorsFile);
-    operation.setCommittedNumOfErrors(errorService.getCommittedNumOfErrors(operationId));
-    operation.setCommittedNumOfWarnings(errorService.getCommittedNumOfWarnings(operationId));
+    marcUpdateService.commitForInstanceMarc(operation);
 
-    if (!FAILED.equals(operation.getStatus())) {
-      operation.setStatus(isEmpty(linkToCommittingErrorsFile) ? COMPLETED : COMPLETED_WITH_ERRORS);
+    if (!INSTANCE_MARC.equals(operation.getEntityType()) || isNull(operation.getLinkToModifiedRecordsMarcFile())) {
+      operation.setProcessedNumOfRecords(operation.getCommittedNumOfRecords());
+      operation.setEndTime(LocalDateTime.now());
+
+      var linkToCommittingErrorsFile = errorService.uploadErrorsToStorage(operationId);
+      operation.setLinkToCommittedRecordsErrorsCsvFile(linkToCommittingErrorsFile);
+      operation.setCommittedNumOfErrors(errorService.getCommittedNumOfErrors(operationId));
+
+      if (!FAILED.equals(operation.getStatus())) {
+        operation.setStatus(isEmpty(linkToCommittingErrorsFile) ? COMPLETED : COMPLETED_WITH_ERRORS);
+      }
+      bulkOperationRepository.save(operation);
     }
-    bulkOperationRepository.save(operation);
   }
 
   private boolean isCurrentTenantNotCentral(String tenantId) {
@@ -658,17 +664,22 @@ public BulkOperation getOperationById(UUID bulkOperationId) {
         yield operation;
       }
       case APPLY_CHANGES -> {
-        if (INSTANCE_MARC.equals(operation.getEntityType())) {
-          var executions = metadataProviderService.getJobExecutions(operation.getDataImportJobProfileId());
-          updateBulkOperationBasedOnDataImportState(executions, operation);
-        } else {
-          var execution = executionRepository.findByBulkOperationId(bulkOperationId);
-          if (execution.isPresent() && StatusType.ACTIVE.equals(execution.get().getStatus())) {
-            operation.setProcessedNumOfRecords(execution.get().getProcessedRecords());
+        var execution = executionRepository.findByBulkOperationId(bulkOperationId);
+        if (execution.isPresent() && StatusType.ACTIVE.equals(execution.get().getStatus())) {
+          var processedNumOfRecords = execution.get().getProcessedRecords();
+          if (INSTANCE_MARC.equals(operation.getEntityType())) {
+            var numOfErrors = operation.getCommittedNumOfErrors() * getProgressMultiplier(operation);
+            operation.setProcessedNumOfRecords(numOfErrors + processedNumOfRecords);
           }
+          operation.setProcessedNumOfRecords(processedNumOfRecords);
         }
         yield bulkOperationRepository.save(operation);
       }
+      case APPLY_MARC_CHANGES -> {
+        var executions = metadataProviderService.getJobExecutions(operation.getDataImportJobProfileId());
+        updateBulkOperationBasedOnDataImportState(executions, operation);
+        yield bulkOperationRepository.save(operation);
+      }
       default -> operation;
     };
   }
@@ -692,8 +703,15 @@ public void processDataImportResult(UUID dataImportJobProfileId) {
   }
 
   private void updateBulkOperationBasedOnDataImportState(List<DataImportJobExecution> executions, BulkOperation operation) {
+    var numOfCommittedAdministrativeUpdates = executionRepository.findAllByBulkOperationId(operation.getId())
+      .stream()
+      .filter(execution -> StatusType.COMPLETED.equals(execution.getStatus()))
+      .map(BulkOperationExecution::getProcessedRecords)
+      .max(Integer::compareTo)
+      .orElse(0);
     var processedNumOfRecords = metadataProviderService.calculateProgress(executions).getCurrent();
-    operation.setProcessedNumOfRecords(operation.getCommittedNumOfErrors() * 2 + processedNumOfRecords);
+    operation.setProcessedNumOfRecords(operation.getCommittedNumOfErrors() * getProgressMultiplier(operation) +
+      numOfCommittedAdministrativeUpdates + processedNumOfRecords);
   }
 
   public BulkOperation getBulkOperationOrThrow(UUID operationId) {
@@ -801,4 +819,11 @@ private synchronized void saveLinks(BulkOperation source) {
   private boolean isCompletedSuccessfully(List<BulkOperationDataProcessing> list) {
     return list.stream().allMatch(p -> StatusType.COMPLETED.equals(p.getStatus()));
   }
+
+  private int getProgressMultiplier(BulkOperation operation) {
+    if (nonNull(operation.getLinkToModifiedRecordsMarcFile())) {
+      return nonNull(operation.getLinkToModifiedRecordsJsonFile()) ? 3 : 2;
+    }
+    return 1;
+  }
 }

src/main/java/org/folio/bulkops/service/ErrorService.java

@@ -10,6 +10,8 @@
 import static org.folio.bulkops.domain.dto.OperationStatusType.REVIEWED_NO_MARC_RECORDS;
 import static org.folio.bulkops.domain.dto.OperationStatusType.REVIEW_CHANGES;
 import static org.folio.bulkops.util.Constants.DATA_IMPORT_ERROR_DISCARDED;
+import static org.folio.bulkops.util.Constants.MSG_NO_ADMINISTRATIVE_CHANGE_REQUIRED;
+import static org.folio.bulkops.util.Constants.MSG_NO_MARC_CHANGE_REQUIRED;
 import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED;
 
 import java.io.ByteArrayInputStream;
@@ -63,7 +65,8 @@ public class ErrorService {
   private final MetadataProviderClient metadataProviderClient;
 
   public void saveError(UUID bulkOperationId, String identifier,  String errorMessage, String uiErrorMessage, String link, ErrorType errorType) {
-    if (MSG_NO_CHANGE_REQUIRED.equals(errorMessage) && executionContentRepository.findFirstByBulkOperationIdAndIdentifier(bulkOperationId, identifier).isPresent()) {
+    if (Set.of(MSG_NO_CHANGE_REQUIRED, MSG_NO_ADMINISTRATIVE_CHANGE_REQUIRED, MSG_NO_MARC_CHANGE_REQUIRED).contains(errorMessage)
+      && executionContentRepository.findFirstByBulkOperationIdAndIdentifier(bulkOperationId, identifier).isPresent()) {
       return;
     }
     executionContentRepository.save(BulkOperationExecutionContent.builder()

src/main/java/org/folio/bulkops/service/MarcUpdateService.java

@@ -2,9 +2,10 @@
 
 import static java.util.Objects.nonNull;
 import static org.folio.bulkops.domain.dto.IdentifierType.HRID;
+import static org.folio.bulkops.domain.dto.OperationStatusType.APPLY_MARC_CHANGES;
 import static org.folio.bulkops.domain.dto.OperationStatusType.FAILED;
 import static org.folio.bulkops.util.Constants.MARC;
-import static org.folio.bulkops.util.Constants.MSG_NO_CHANGE_REQUIRED;
+import static org.folio.bulkops.util.Constants.MSG_NO_MARC_CHANGE_REQUIRED;
 import static org.folio.bulkops.util.MarcHelper.fetchInstanceUuidOrElseHrid;
 
 import com.fasterxml.jackson.core.JsonFactory;
@@ -36,7 +37,7 @@
 @Log4j2
 @RequiredArgsConstructor
 public class MarcUpdateService {
-  public static final String CHANGED_MARC_PATH_TEMPLATE = "%s/%s-Changed-Records-%s.mrc";
+  public static final String CHANGED_MARC_PATH_TEMPLATE = "%s/%s-Changed-Records-MARC-%s.mrc";
   public static final String MSG_BULK_EDIT_SUPPORTED_FOR_MARC_ONLY = "Bulk edit of MARC fields is supported only for instance with MARC source.";
 
   private final BulkOperationExecutionRepository executionRepository;
@@ -49,8 +50,8 @@ public class MarcUpdateService {
   @Transactional
   public void commitForInstanceMarc(BulkOperation bulkOperation) {
     if (StringUtils.isNotEmpty(bulkOperation.getLinkToModifiedRecordsMarcFile())) {
-      bulkOperation.setTotalNumOfRecords(bulkOperation.getTotalNumOfRecords() * 2);
-      bulkOperation.setProcessedNumOfRecords(bulkOperation.getProcessedNumOfRecords() * 2);
+      bulkOperation.setStatus(APPLY_MARC_CHANGES);
+      bulkOperationRepository.save(bulkOperation);
 
       var execution = executionRepository.save(BulkOperationExecution.builder()
         .bulkOperationId(bulkOperation.getId())
@@ -104,7 +105,7 @@ private String prepareCommittedFile(BulkOperation bulkOperation) throws IOExcept
           var identifier = HRID.equals(bulkOperation.getIdentifierType()) ?
             originalRecord.getControlNumber() :
             fetchInstanceUuidOrElseHrid(originalRecord);
-          errorService.saveError(bulkOperation.getId(), identifier, MSG_NO_CHANGE_REQUIRED, ErrorType.WARNING);
+          errorService.saveError(bulkOperation.getId(), identifier, MSG_NO_MARC_CHANGE_REQUIRED, ErrorType.WARNING);
         } else {
           MarcDateHelper.updateDateTimeControlField(modifiedRecord, currentDate);
           writerForResultMarcFile.writeRecord(modifiedRecord);