diff --git a/NEWS.md b/NEWS.md index 7f1f6cb7b..ddcfd0f60 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,5 +1,21 @@ ## 20.2.5 2024-05-24 -* Don't group fields with same tags together ([MODINV-1026](https://issues.folio.org/browse/MODINV-1026)) +* [MODINV-1026](https://folio-org.atlassian.net/browse/MODINV-1026) Don't group fields with same tags together + +## 20.2.4 2024-05-09 +* [MODINV-1024](https://folio-org.atlassian.net/browse/MODINV-1024) Fix error while updating Instances and Items with electronic access without URI field populated. + +## 20.2.3 2024-04-25 +* [MODINV-1022](https://folio-org.atlassian.net/browse/MODINV-1022) Circulation History in Item record removed when Item is marked Missing/Long missing/ Unavailable + +## 20.2.2 2024-04-22 +* [MODINV-1009](https://folio-org.atlassian.net/browse/MODINV-1009) Circulation History in Item record removed when Item is marked Missing/Long missing/ Unavailable +* [MODINV-1012](https://folio-org.atlassian.net/browse/MODINV-1012) Invalid values (as it is) created in JSON when value is not matching accepted options provided in Instance field mapping for Nature of Content term + +## 20.2.1 2024-04-12 +* [MODINV-1001](https://folio-org.atlassian.net/browse/MODINV-1001) Fix sorting for Items on Instance details page +* [MODINV-999](https://folio-org.atlassian.net/browse/MODINV-999)"PMSystem" displayed as source in "quickmarc" view when record was created by "Non-matches" action of job profile +* [MODINV-1003](https://folio-org.atlassian.net/browse/MODINV-1003) The result table is not displayed in the file details log +* [MODINV-997](https://folio-org.atlassian.net/browse/MODINV-997) Keep order of MARC fields while Creating/Deriving/Editing MARC records ## 20.2.0 2023-03-20 * Inventory cannot process Holdings with virtual fields ([MODINV-941](https://issues.folio.org/browse/MODINV-941)) diff --git a/pom.xml b/pom.xml index b90dfce90..d137cfee1 100644 --- a/pom.xml +++ b/pom.xml @@ -2,7 +2,7 @@ 4.0.0 mod-inventory org.folio - 20.2.1-SNAPSHOT + 20.2.5-SNAPSHOT Apache License 2.0 diff --git a/src/main/java/org/folio/inventory/dataimport/consumers/DataImportKafkaHandler.java b/src/main/java/org/folio/inventory/dataimport/consumers/DataImportKafkaHandler.java index 3c4044abd..61df80316 100644 --- a/src/main/java/org/folio/inventory/dataimport/consumers/DataImportKafkaHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/consumers/DataImportKafkaHandler.java @@ -91,6 +91,7 @@ public class DataImportKafkaHandler implements AsyncRecordHandler handle(KafkaConsumerRecord record) { Map headersMap = KafkaHeaderUtils.kafkaHeadersToMap(record.headers()); String recordId = headersMap.get(RECORD_ID_HEADER); String chunkId = headersMap.get(CHUNK_ID_HEADER); + String userId = headersMap.get(USER_ID_HEADER); String jobExecutionId = eventPayload.getJobExecutionId(); LOGGER.info("Data import event payload has been received with event type: {}, recordId: {} by jobExecution: {} and chunkId: {}", eventPayload.getEventType(), recordId, jobExecutionId, chunkId); eventPayload.getContext().put(RECORD_ID_HEADER, recordId); eventPayload.getContext().put(CHUNK_ID_HEADER, chunkId); + eventPayload.getContext().put(USER_ID_HEADER, userId); Context context = EventHandlingUtil.constructContext(eventPayload.getTenant(), eventPayload.getToken(), eventPayload.getOkapiUrl()); String jobProfileSnapshotId = eventPayload.getContext().get(PROFILE_SNAPSHOT_ID_KEY); diff --git a/src/main/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandler.java b/src/main/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandler.java index 3456cb5da..b76e331d0 100644 --- a/src/main/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandler.java @@ -15,6 +15,7 @@ import org.folio.inventory.dataimport.services.OrderHelperService; import org.folio.inventory.dataimport.util.AdditionalFieldsUtil; import org.folio.inventory.dataimport.util.ParsedRecordUtil; +import org.folio.inventory.dataimport.util.ValidationUtil; import org.folio.inventory.domain.instances.Instance; import org.folio.inventory.domain.instances.InstanceCollection; import org.folio.inventory.domain.relationship.RecordToEntity; @@ -41,8 +42,7 @@ import static org.folio.ActionProfile.FolioRecord.MARC_BIBLIOGRAPHIC; import static org.folio.DataImportEventTypes.DI_INVENTORY_INSTANCE_CREATED; import static org.folio.DataImportEventTypes.DI_INVENTORY_INSTANCE_CREATED_READY_FOR_POST_PROCESSING; -import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.SUBFIELD_I; -import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_999; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.*; import static org.folio.inventory.dataimport.util.DataImportConstants.UNIQUE_ID_ERROR_MESSAGE; import static org.folio.inventory.dataimport.util.LoggerUtil.logParametersEventHandler; import static org.folio.inventory.domain.instances.Instance.HRID_KEY; @@ -93,6 +93,7 @@ public CompletableFuture handle(DataImportEventPayload d Context context = EventHandlingUtil.constructContext(dataImportEventPayload.getTenant(), dataImportEventPayload.getToken(), dataImportEventPayload.getOkapiUrl()); Record targetRecord = Json.decodeValue(payloadContext.get(EntityType.MARC_BIBLIOGRAPHIC.value()), Record.class); + var sourceContent = targetRecord.getParsedRecord().getContent().toString(); if (!Boolean.parseBoolean(payloadContext.get("acceptInstanceId")) && AdditionalFieldsUtil.getValue(targetRecord, TAG_999, SUBFIELD_I).isPresent()) { LOGGER.error(INSTANCE_CREATION_999_ERROR_MESSAGE); @@ -118,19 +119,33 @@ public CompletableFuture handle(DataImportEventPayload d .compose(v -> { InstanceCollection instanceCollection = storage.getInstanceCollection(context); JsonObject instanceAsJson = prepareInstance(dataImportEventPayload, instanceId, jobExecutionId); - List errors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields); - if (!errors.isEmpty()) { - String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", errors, + List requiredFieldsErrors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields); + if (!requiredFieldsErrors.isEmpty()) { + String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", requiredFieldsErrors, jobExecutionId, recordId, chunkId); LOGGER.warn(msg); return Future.failedFuture(msg); } Instance mappedInstance = Instance.fromJson(instanceAsJson); + + List invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance); + if (!invalidUUIDsErrors.isEmpty()) { + String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors, + jobExecutionId, recordId, chunkId); + LOGGER.warn(msg); + return Future.failedFuture(msg); + } + return addInstance(mappedInstance, instanceCollection) .compose(createdInstance -> getPrecedingSucceedingTitlesHelper().createPrecedingSucceedingTitles(mappedInstance, context).map(createdInstance)) .compose(createdInstance -> executeFieldsManipulation(createdInstance, targetRecord)) - .compose(createdInstance -> saveRecordInSrsAndHandleResponse(dataImportEventPayload, targetRecord, createdInstance, instanceCollection, dataImportEventPayload.getTenant())); + .compose(createdInstance -> { + var targetContent = targetRecord.getParsedRecord().getContent().toString(); + var content = reorderMarcRecordFields(sourceContent, targetContent); + targetRecord.setParsedRecord(targetRecord.getParsedRecord().withContent(content)); + return saveRecordInSrsAndHandleResponse(dataImportEventPayload, targetRecord, createdInstance, instanceCollection, dataImportEventPayload.getTenant()); + }); }) .onSuccess(ar -> { dataImportEventPayload.getContext().put(INSTANCE.value(), Json.encode(ar)); diff --git a/src/main/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandler.java b/src/main/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandler.java index a60c40d22..bfc53b753 100644 --- a/src/main/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandler.java +++ b/src/main/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandler.java @@ -17,6 +17,7 @@ import org.folio.inventory.dataimport.cache.MappingMetadataCache; import org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil; import org.folio.inventory.dataimport.util.AdditionalFieldsUtil; +import org.folio.inventory.dataimport.util.ValidationUtil; import org.folio.inventory.domain.instances.Instance; import org.folio.inventory.domain.instances.InstanceCollection; import org.folio.inventory.exceptions.NotFoundException; @@ -194,6 +195,14 @@ private void processInstanceUpdate(DataImportEventPayload dataImportEventPayload org.folio.rest.jaxrs.model.Record targetRecord = Json.decodeValue(marcBibAsJson, org.folio.rest.jaxrs.model.Record.class); Instance mappedInstance = Instance.fromJson(instanceAsJson); + List invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance); + if (!invalidUUIDsErrors.isEmpty()) { + String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors, + jobExecutionId, recordId, chunkId); + LOGGER.warn(msg); + return Future.failedFuture(msg); + } + return updateInstanceAndRetryIfOlExists(mappedInstance, instanceCollection, dataImportEventPayload) .compose(updatedInstance -> getPrecedingSucceedingTitlesHelper().getExistingPrecedingSucceedingTitles(mappedInstance, context)) .map(precedingSucceedingTitles -> precedingSucceedingTitles.stream() diff --git a/src/main/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtil.java b/src/main/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtil.java index e45d7183c..7eebf04f6 100644 --- a/src/main/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtil.java +++ b/src/main/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtil.java @@ -1,10 +1,14 @@ package org.folio.inventory.dataimport.util; +import static java.lang.String.format; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.apache.commons.lang3.StringUtils.isNotBlank; +import static org.apache.commons.lang3.StringUtils.isNotEmpty; + import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang3.tuple.Pair; import com.github.benmanes.caffeine.cache.CacheLoader; import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.LoadingCache; @@ -21,17 +25,13 @@ import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; -import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.ForkJoinPool; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.folio.inventory.domain.instances.Instance; @@ -51,44 +51,28 @@ import org.marc4j.marc.Subfield; import org.marc4j.marc.VariableField; -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.nio.charset.StandardCharsets; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.List; -import java.util.Optional; -import java.util.concurrent.ForkJoinPool; - -import static java.lang.String.format; -import static org.apache.commons.lang3.StringUtils.isBlank; -import static org.apache.commons.lang3.StringUtils.isNotBlank; -import static org.apache.commons.lang3.StringUtils.isNotEmpty; - /** * Util to work with additional fields */ public final class AdditionalFieldsUtil { - private static final Logger LOGGER = LogManager.getLogger(); public static final DateTimeFormatter dateTime005Formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss.S"); public static final String TAG_005 = "005"; public static final String TAG_999 = "999"; public static final String TAG_001 = "001"; + public static final char SUBFIELD_I = 'i'; + public static final String FIELDS = "fields"; + private static final Logger LOGGER = LogManager.getLogger(); private static final String TAG_003 = "003"; private static final String TAG_035 = "035"; private static final char TAG_035_SUB = 'a'; private static final char TAG_035_IND = ' '; private static final String ANY_STRING = "*"; private static final char INDICATOR = 'f'; - public static final char SUBFIELD_I = 'i'; private static final String HR_ID_FIELD = "hrid"; - public static final String FIELDS = "fields"; - private static final ObjectMapper objectMapper = new ObjectMapper(); private static final CacheLoader parsedRecordContentCacheLoader; private static final LoadingCache parsedRecordContentCache; + private static final ObjectMapper objectMapper = new ObjectMapper(); static { // this function is executed when creating a new item to be saved in the cache. @@ -132,11 +116,6 @@ public final class AdditionalFieldsUtil { private AdditionalFieldsUtil() { } - @FunctionalInterface - public interface AddControlledFieldToMarcRecordFunction { - void apply(String field, String value, org.marc4j.marc.Record marcRecord); - } - public static CacheStats getCacheStats() { return parsedRecordContentCache.stats(); } @@ -144,16 +123,17 @@ public static CacheStats getCacheStats() { /** * Adds field if it does not exist and a subfield with a value to that field * - * @param recordForUpdate record that needs to be updated - * @param field field that should contain new subfield - * @param subfield new subfield to add - * @param value value of the subfield to add + * @param recordForUpdate record that needs to be updated + * @param field field that should contain new subfield + * @param subfield new subfield to add + * @param value value of the subfield to add * @return true if succeeded, false otherwise */ public static boolean addFieldToMarcRecord(Record recordForUpdate, String field, char subfield, String value) { boolean result = false; try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { - if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null && recordForUpdate.getParsedRecord().getContent() != null) { + if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null + && recordForUpdate.getParsedRecord().getContent() != null) { MarcWriter streamWriter = new MarcStreamWriter(new ByteArrayOutputStream()); MarcJsonWriter jsonWriter = new MarcJsonWriter(os); MarcFactory factory = MarcFactory.newInstance(); @@ -162,8 +142,8 @@ public static boolean addFieldToMarcRecord(Record recordForUpdate, String field, VariableField variableField = getSingleFieldByIndicators(marcRecord.getVariableFields(field)); DataField dataField; if (variableField != null - && ((DataField) variableField).getIndicator1() == INDICATOR - && ((DataField) variableField).getIndicator2() == INDICATOR + && ((DataField) variableField).getIndicator1() == INDICATOR + && ((DataField) variableField).getIndicator2() == INDICATOR ) { dataField = (DataField) variableField; marcRecord.removeVariableField(variableField); @@ -194,7 +174,7 @@ public static boolean addFieldToMarcRecord(Record recordForUpdate, String field, /** * Updates field 005 for case when this field is not protected. * - * @param recordForUpdate record to update + * @param recordForUpdate record to update * @param mappingParameters mapping parameters */ public static void updateLatestTransactionDate(Record recordForUpdate, MappingParameters mappingParameters) { @@ -213,15 +193,16 @@ public static void updateLatestTransactionDate(Record recordForUpdate, MappingPa * Adds new controlled field to marc record * * @param recordForUpdate record that needs to be updated - * @param field tag of controlled field - * @param value value of the field to add + * @param field tag of controlled field + * @param value value of the field to add * @return true if succeeded, false otherwise */ public static boolean addControlledFieldToMarcRecord(Record recordForUpdate, String field, String value, AddControlledFieldToMarcRecordFunction addFieldFunc) { boolean result = false; try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { - if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null && recordForUpdate.getParsedRecord().getContent() != null) { + if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null + && recordForUpdate.getParsedRecord().getContent() != null) { MarcWriter streamWriter = new MarcStreamWriter(new ByteArrayOutputStream()); MarcJsonWriter jsonWriter = new MarcJsonWriter(os); @@ -251,8 +232,9 @@ public static void addControlledFieldToMarcRecord(String field, String value, or marcRecord.addVariableField(dataField); } - public static void replaceOrAddControlledFieldInMarcRecord(String field, String value, org.marc4j.marc.Record marcRecord) { - var currentField = (ControlField) marcRecord.getVariableField(field); + public static void replaceOrAddControlledFieldInMarcRecord(String field, String value, + org.marc4j.marc.Record marcRecord) { + var currentField = (ControlField) marcRecord.getVariableField(field); var newControlField = MarcFactory.newInstance().newControlField(field, value); if (currentField != null) { marcRecord.getControlFields().set(marcRecord.getControlFields().indexOf(currentField), newControlField); @@ -292,7 +274,7 @@ public static void fill001FieldInMarcRecord(Record marcRecord, String hrId) { * Read value from controlled field in marc record * * @param srcRecord marc record - * @param tag tag to read + * @param tag tag to read * @return value from field */ public static String getValueFromControlledField(Record srcRecord, String tag) { @@ -308,131 +290,35 @@ public static String getValueFromControlledField(Record srcRecord, String tag) { } } } catch (Exception e) { - LOGGER.warn("getValueFromControlledField:: Failed to read controlled field {} from record {}", tag, srcRecord.getId(), e); + LOGGER.warn("getValueFromControlledField:: Failed to read controlled field {} from record {}", tag, + srcRecord.getId(), e); return null; } return null; } public static Optional getValue(Record srcRecord, String tag, char subfield) { - return Optional.ofNullable(computeMarcRecord(srcRecord)) - .stream() - .flatMap(marcRecord -> marcRecord.getVariableFields(tag).stream()) - .flatMap(field -> getFieldValue(field, subfield).stream()) - .findFirst(); - } - - private static Optional getFieldValue(VariableField field, char subfield) { - if (field instanceof DataField dataField) { - return dataField.getSubfields(subfield).stream().findFirst().map(Subfield::getData); - } else if (field instanceof ControlField controlField) { - return Optional.ofNullable(controlField.getData()); - } else { - return Optional.empty(); - } - } - - private static MarcReader buildMarcReader(Record srcRecord) { - String content = normalizeContent(srcRecord.getParsedRecord()); - return new MarcJsonReader(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); - } - - private static VariableField getSingleFieldByIndicators(List list) { - if (CollectionUtils.isEmpty(list)) { - return null; - } - return list.stream() - .filter(f -> ((DataField) f).getIndicator1() == INDICATOR && ((DataField) f).getIndicator2() == INDICATOR) - .findFirst() - .orElse(null); - } - - /** - * Checks whether field 005 needs to be updated or this field is protected. - * - * @param srcRecord record to check - * @param mappingParameters mapping parameters - * @return true for case when field 005 have to updated - */ - private static boolean isField005NeedToUpdate(Record srcRecord, MappingParameters mappingParameters) { - boolean needToUpdate = true; - List fieldProtectionSettings = mappingParameters.getMarcFieldProtectionSettings(); - if (CollectionUtils.isNotEmpty(fieldProtectionSettings)) { - MarcReader reader = new MarcJsonReader(new ByteArrayInputStream(srcRecord.getParsedRecord().getContent().toString().getBytes())); - if (reader.hasNext()) { - org.marc4j.marc.Record marcRecord = reader.next(); - List variableFields = marcRecord.getVariableFields(TAG_005); - if(!variableFields.isEmpty()) { - VariableField field = variableFields.get(0); - needToUpdate = isNotProtected(fieldProtectionSettings, (ControlField) field); - } - } - } - return needToUpdate; - } - - /** - * Checks is the control field is protected or not. - * - * @param fieldProtectionSettings List of MarcFieldProtectionSettings - * @param field Control field that is being checked - * @return true for case when control field isn't protected - */ - private static boolean isNotProtected(List fieldProtectionSettings, ControlField field) { - return fieldProtectionSettings.stream() - .filter(setting -> setting.getField().equals(ANY_STRING) || setting.getField().equals(field.getTag())) - .noneMatch(setting -> setting.getData().equals(ANY_STRING) || setting.getData().equals(field.getData())); - } - - private static org.marc4j.marc.Record computeMarcRecord(Record srcRecord) { - if (srcRecord != null && srcRecord.getParsedRecord() != null && isNotBlank(srcRecord.getParsedRecord().getContent().toString())) { - try { - var content = normalizeContent(srcRecord.getParsedRecord().getContent()); - return parsedRecordContentCache.get(content); - } catch (Exception e) { - LOGGER.warn("computeMarcRecord:: Error during the transformation to marc record", e); - try { - MarcReader reader = buildMarcReader(srcRecord); - if (reader.hasNext()) { - return reader.next(); - } - } catch (Exception ex) { - LOGGER.warn("computeMarcRecord:: Error during the building of MarcReader", ex); - } - return null; - } - } - return null; - } - - /** - * Normalize parsed record content of {@link ParsedRecord} is type {@link String} - * - * @param parsedRecord parsed record - * @return parsed record normalized content - */ - private static String normalizeContent(ParsedRecord parsedRecord) { - Object content = parsedRecord.getContent(); - return (content instanceof String contentStr ? new JsonObject(contentStr) : JsonObject.mapFrom(content)).encode(); - } - - private static String normalizeContent(Object content) { - return content instanceof String contentStr ? contentStr : Json.encode(content); + return Optional.ofNullable(computeMarcRecord(srcRecord)) + .stream() + .flatMap(marcRecord -> marcRecord.getVariableFields(tag).stream()) + .flatMap(field -> getFieldValue(field, subfield).stream()) + .findFirst(); } /** * Remove field from marc record * - * @param recordForUpdate record that needs to be updated - * @param fieldName tag of the field - * @param subfield subfield of the field - * @param value value of the field + * @param recordForUpdate record that needs to be updated + * @param fieldName tag of the field + * @param subfield subfield of the field + * @param value value of the field * @return true if succeeded, false otherwise */ public static boolean removeField(Record recordForUpdate, String fieldName, char subfield, String value) { boolean isFieldRemoveSucceed = false; try (ByteArrayOutputStream baos = new ByteArrayOutputStream()) { - if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null && recordForUpdate.getParsedRecord().getContent() != null) { + if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null + && recordForUpdate.getParsedRecord().getContent() != null) { MarcWriter marcStreamWriter = new MarcStreamWriter(new ByteArrayOutputStream()); MarcJsonWriter marcJsonWriter = new MarcJsonWriter(baos); org.marc4j.marc.Record marcRecord = computeMarcRecord(recordForUpdate); @@ -462,55 +348,11 @@ public static boolean removeField(Record recordForUpdate, String fieldName, char return isFieldRemoveSucceed; } - private static boolean removeFieldByNameAndValue(org.marc4j.marc.Record marcRecord, String fieldName, char subfield, String value) { - boolean isFieldFound = false; - List variableFields = marcRecord.getVariableFields(fieldName); - for (VariableField variableField : variableFields) { - if (isFieldContainsValue(variableField, subfield, value)) { - marcRecord.removeVariableField(variableField); - isFieldFound = true; - break; - } - } - return isFieldFound; - } - - private static boolean removeFirstFoundFieldByName(org.marc4j.marc.Record marcRecord, String fieldName) { - boolean isFieldFound = false; - VariableField variableField = marcRecord.getVariableField(fieldName); - if (variableField != null) { - marcRecord.removeVariableField(variableField); - isFieldFound = true; - } - return isFieldFound; - } - - /** - * Checks if the field contains a certain value in the selected subfield - * - * @param field from MARC BIB record - * @param subfield subfield of the field - * @param value value of the field - * @return true if contains, false otherwise - */ - private static boolean isFieldContainsValue(VariableField field, char subfield, String value) { - boolean isContains = false; - if (field instanceof DataField) { - for (Subfield sub : ((DataField) field).getSubfields(subfield)) { - if (isNotEmpty(sub.getData()) && sub.getData().contains(value.trim())) { - isContains = true; - break; - } - } - } - return isContains; - } - /** * remove field from marc record * * @param recordForUpdate record that needs to be updated - * @param field tag of the field + * @param field tag of the field * @return true if succeeded, false otherwise */ public static boolean removeField(Record recordForUpdate, String field) { @@ -520,8 +362,8 @@ public static boolean removeField(Record recordForUpdate, String field) { /** * Check if record should be filled with specific fields. * - * @param srcRecord - source record. - * @param instance - instance. + * @param srcRecord - source record. + * @param instance - instance. * @return - true if filling needed. */ public static boolean isFieldsFillingNeeded(Record srcRecord, Instance instance) { @@ -530,22 +372,20 @@ public static boolean isFieldsFillingNeeded(Record srcRecord, Instance instance) externalIdsHolder.getInstanceId(), externalIdsHolder.getInstanceHrid()); } - private static boolean isValidIdAndHrid(String id, String hrid, String externalId, String externalHrid) { - return (isNotEmpty(externalId) && isNotEmpty(externalHrid)) && (id.equals(externalId) && !hrid.equals(externalHrid)); - } - /** * Adds new data field to marc record * * @param recordForUpdate record that needs to be updated - * @param tag tag of data field - * @param value value of the field to add + * @param tag tag of data field + * @param value value of the field to add * @return true if succeeded, false otherwise */ - public static boolean addDataFieldToMarcRecord(Record recordForUpdate, String tag, char ind1, char ind2, char subfield, String value) { + public static boolean addDataFieldToMarcRecord(Record recordForUpdate, String tag, char ind1, char ind2, + char subfield, String value) { boolean result = false; try (ByteArrayOutputStream os = new ByteArrayOutputStream()) { - if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null && recordForUpdate.getParsedRecord().getContent() != null) { + if (recordForUpdate != null && recordForUpdate.getParsedRecord() != null + && recordForUpdate.getParsedRecord().getContent() != null) { MarcWriter streamWriter = new MarcStreamWriter(new ByteArrayOutputStream()); MarcJsonWriter jsonWriter = new MarcJsonWriter(os); MarcFactory factory = MarcFactory.newInstance(); @@ -572,18 +412,6 @@ public static boolean addDataFieldToMarcRecord(Record recordForUpdate, String ta return result; } - private static void addDataFieldInNumericalOrder(DataField field, org.marc4j.marc.Record marcRecord) { - String tag = field.getTag(); - List dataFields = marcRecord.getDataFields(); - for (int i = 0; i < dataFields.size(); i++) { - if (dataFields.get(i).getTag().compareTo(tag) > 0) { - marcRecord.getDataFields().add(i, field); - return; - } - } - marcRecord.addVariableField(field); - } - public static String mergeFieldsFor035(String valueFrom003, String valueFrom001) { if (isBlank(valueFrom003)) { return valueFrom001; @@ -595,8 +423,8 @@ public static String mergeFieldsFor035(String valueFrom003, String valueFrom001) * Check if data field with the same value exist * * @param recordForUpdate record that needs to be updated - * @param tag tag of data field - * @param value value of the field to add + * @param tag tag of data field + * @param value value of the field to add * @return true if exist */ public static boolean isFieldExist(Record recordForUpdate, String tag, char subfield, String value) { @@ -611,8 +439,8 @@ public static boolean isFieldExist(Record recordForUpdate, String tag, char subf } } } else if (field instanceof ControlField controlField - && isNotEmpty(controlField.getData()) - && ((ControlField) field).getData().equals(value.trim())) { + && isNotEmpty(controlField.getData()) + && ((ControlField) field).getData().equals(value.trim())) { return true; } } @@ -624,10 +452,6 @@ && isNotEmpty(controlField.getData()) return false; } - private static String getRecordId(Record srcRecord) { - return srcRecord != null ? srcRecord.getId() : ""; - } - public static void remove035FieldWhenRecordContainsHrId(Record srcRecord) { if (Record.RecordType.MARC_BIB.equals(srcRecord.getRecordType())) { String hrid = getValueFromControlledField(srcRecord, TAG_001); @@ -649,15 +473,18 @@ public static void fillHrIdFieldInMarcRecord(Pair recordInst String valueFrom001 = AdditionalFieldsUtil.getValueFromControlledField(recordInstancePair.getKey(), TAG_001); if (!StringUtils.equals(hrid, valueFrom001)) { if (StringUtils.isNotEmpty(valueFrom001)) { - String originalHrIdPrefix = AdditionalFieldsUtil.getValueFromControlledField(recordInstancePair.getKey(), TAG_003); + String originalHrIdPrefix = + AdditionalFieldsUtil.getValueFromControlledField(recordInstancePair.getKey(), TAG_003); String originalHrId = AdditionalFieldsUtil.mergeFieldsFor035(originalHrIdPrefix, valueFrom001); if (!AdditionalFieldsUtil.isFieldExist(recordInstancePair.getKey(), TAG_035, TAG_035_SUB, originalHrId)) { - AdditionalFieldsUtil.addDataFieldToMarcRecord(recordInstancePair.getKey(), TAG_035, TAG_035_IND, TAG_035_IND, TAG_035_SUB, originalHrId); + AdditionalFieldsUtil.addDataFieldToMarcRecord(recordInstancePair.getKey(), TAG_035, TAG_035_IND, TAG_035_IND, + TAG_035_SUB, originalHrId); } } AdditionalFieldsUtil.removeField(recordInstancePair.getKey(), TAG_001); if (StringUtils.isNotEmpty(hrid)) { - AdditionalFieldsUtil.addControlledFieldToMarcRecord(recordInstancePair.getKey(), TAG_001, hrid, AdditionalFieldsUtil::addControlledFieldToMarcRecord); + AdditionalFieldsUtil.addControlledFieldToMarcRecord(recordInstancePair.getKey(), TAG_001, hrid, + AdditionalFieldsUtil::addControlledFieldToMarcRecord); } } else { AdditionalFieldsUtil.remove035WithActualHrId(recordInstancePair.getKey(), hrid); @@ -709,6 +536,172 @@ public static String reorderMarcRecordFields(String sourceOrderContent, String s } } + private static Optional getFieldValue(VariableField field, char subfield) { + if (field instanceof DataField dataField) { + return dataField.getSubfields(subfield).stream().findFirst().map(Subfield::getData); + } else if (field instanceof ControlField controlField) { + return Optional.ofNullable(controlField.getData()); + } else { + return Optional.empty(); + } + } + + private static MarcReader buildMarcReader(Record srcRecord) { + String content = normalizeContent(srcRecord.getParsedRecord()); + return new MarcJsonReader(new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8))); + } + + private static VariableField getSingleFieldByIndicators(List list) { + if (CollectionUtils.isEmpty(list)) { + return null; + } + return list.stream() + .filter(f -> ((DataField) f).getIndicator1() == INDICATOR && ((DataField) f).getIndicator2() == INDICATOR) + .findFirst() + .orElse(null); + } + + /** + * Checks whether field 005 needs to be updated or this field is protected. + * + * @param srcRecord record to check + * @param mappingParameters mapping parameters + * @return true for case when field 005 have to updated + */ + private static boolean isField005NeedToUpdate(Record srcRecord, MappingParameters mappingParameters) { + boolean needToUpdate = true; + List fieldProtectionSettings = mappingParameters.getMarcFieldProtectionSettings(); + if (CollectionUtils.isNotEmpty(fieldProtectionSettings)) { + MarcReader reader = + new MarcJsonReader(new ByteArrayInputStream(srcRecord.getParsedRecord().getContent().toString().getBytes())); + if (reader.hasNext()) { + org.marc4j.marc.Record marcRecord = reader.next(); + List variableFields = marcRecord.getVariableFields(TAG_005); + if (!variableFields.isEmpty()) { + VariableField field = variableFields.get(0); + needToUpdate = isNotProtected(fieldProtectionSettings, (ControlField) field); + } + } + } + return needToUpdate; + } + + /** + * Checks is the control field is protected or not. + * + * @param fieldProtectionSettings List of MarcFieldProtectionSettings + * @param field Control field that is being checked + * @return true for case when control field isn't protected + */ + private static boolean isNotProtected(List fieldProtectionSettings, ControlField field) { + return fieldProtectionSettings.stream() + .filter(setting -> setting.getField().equals(ANY_STRING) || setting.getField().equals(field.getTag())) + .noneMatch(setting -> setting.getData().equals(ANY_STRING) || setting.getData().equals(field.getData())); + } + + private static org.marc4j.marc.Record computeMarcRecord(Record srcRecord) { + if (srcRecord != null && srcRecord.getParsedRecord() != null && isNotBlank( + srcRecord.getParsedRecord().getContent().toString())) { + try { + var content = normalizeContent(srcRecord.getParsedRecord().getContent()); + return parsedRecordContentCache.get(content); + } catch (Exception e) { + LOGGER.warn("computeMarcRecord:: Error during the transformation to marc record", e); + try { + MarcReader reader = buildMarcReader(srcRecord); + if (reader.hasNext()) { + return reader.next(); + } + } catch (Exception ex) { + LOGGER.warn("computeMarcRecord:: Error during the building of MarcReader", ex); + } + return null; + } + } + return null; + } + + /** + * Normalize parsed record content of {@link ParsedRecord} is type {@link String} + * + * @param parsedRecord parsed record + * @return parsed record normalized content + */ + private static String normalizeContent(ParsedRecord parsedRecord) { + Object content = parsedRecord.getContent(); + return (content instanceof String contentStr ? new JsonObject(contentStr) : JsonObject.mapFrom(content)).encode(); + } + + private static String normalizeContent(Object content) { + return content instanceof String contentStr ? contentStr : Json.encode(content); + } + + private static boolean removeFieldByNameAndValue(org.marc4j.marc.Record marcRecord, String fieldName, char subfield, + String value) { + boolean isFieldFound = false; + List variableFields = marcRecord.getVariableFields(fieldName); + for (VariableField variableField : variableFields) { + if (isFieldContainsValue(variableField, subfield, value)) { + marcRecord.removeVariableField(variableField); + isFieldFound = true; + break; + } + } + return isFieldFound; + } + + private static boolean removeFirstFoundFieldByName(org.marc4j.marc.Record marcRecord, String fieldName) { + boolean isFieldFound = false; + VariableField variableField = marcRecord.getVariableField(fieldName); + if (variableField != null) { + marcRecord.removeVariableField(variableField); + isFieldFound = true; + } + return isFieldFound; + } + + /** + * Checks if the field contains a certain value in the selected subfield + * + * @param field from MARC BIB record + * @param subfield subfield of the field + * @param value value of the field + * @return true if contains, false otherwise + */ + private static boolean isFieldContainsValue(VariableField field, char subfield, String value) { + boolean isContains = false; + if (field instanceof DataField) { + for (Subfield sub : ((DataField) field).getSubfields(subfield)) { + if (isNotEmpty(sub.getData()) && sub.getData().contains(value.trim())) { + isContains = true; + break; + } + } + } + return isContains; + } + + private static boolean isValidIdAndHrid(String id, String hrid, String externalId, String externalHrid) { + return (isNotEmpty(externalId) && isNotEmpty(externalHrid)) && (id.equals(externalId) && !hrid.equals( + externalHrid)); + } + + private static void addDataFieldInNumericalOrder(DataField field, org.marc4j.marc.Record marcRecord) { + String tag = field.getTag(); + List dataFields = marcRecord.getDataFields(); + for (int i = 0; i < dataFields.size(); i++) { + if (dataFields.get(i).getTag().compareTo(tag) > 0) { + marcRecord.getDataFields().add(i, field); + return; + } + } + marcRecord.addVariableField(field); + } + + private static String getRecordId(Record srcRecord) { + return srcRecord != null ? srcRecord.getId() : ""; + } + private static List toNodeList(ArrayNode fieldsArrayNode) { var nodes = new LinkedList(); for (var node : fieldsArrayNode) { @@ -760,4 +753,9 @@ private static List getSourceFields(String source) { } return sourceFields; } + + @FunctionalInterface + public interface AddControlledFieldToMarcRecordFunction { + void apply(String field, String value, org.marc4j.marc.Record marcRecord); + } } diff --git a/src/main/java/org/folio/inventory/dataimport/util/ValidationUtil.java b/src/main/java/org/folio/inventory/dataimport/util/ValidationUtil.java new file mode 100644 index 000000000..0f694302c --- /dev/null +++ b/src/main/java/org/folio/inventory/dataimport/util/ValidationUtil.java @@ -0,0 +1,48 @@ +package org.folio.inventory.dataimport.util; + +import org.folio.inventory.domain.instances.Instance; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +/** + * Util for detailed validation different entities. + */ +public class ValidationUtil { + + private ValidationUtil() { + } + + /** + * Validate fields inside the Instance entity. Validation based on checking if specific fields were mapped as UUIDs. + * If not - then the list with errors will be returned. + * Example: "Value 'invalid not UUID value' is not a UUID for someFieldName field" + * @param instance target Instance for validation + * @return ArrayList with errors when the needed fields are NOT as UUID. + */ + public static List validateUUIDs(Instance instance) { + ArrayList errorMessages = new ArrayList<>(); + + //TODO: This will be extended for different fields and entities.That's why there are so many methods just for 1 field. + // Branch for it extending validation: MODINV-1012-extended + validateField(errorMessages, instance.getNatureOfContentTermIds(), "natureOfContentTermIds"); + + return errorMessages; + } + + private static void validateField(List errorMessages, List values, String fieldName) { + values.stream() + .filter(value -> !isUUID(value)) + .forEach(value -> errorMessages.add(String.format("Value '%s' is not a UUID for %s field", value, fieldName))); + } + + private static boolean isUUID(String value) { + try { + UUID.fromString(value); + return true; + } catch (Exception ex) { + return false; + } + } +} diff --git a/src/main/java/org/folio/inventory/domain/instances/Instance.java b/src/main/java/org/folio/inventory/domain/instances/Instance.java index 4fa289808..e04026364 100644 --- a/src/main/java/org/folio/inventory/domain/instances/Instance.java +++ b/src/main/java/org/folio/inventory/domain/instances/Instance.java @@ -26,6 +26,7 @@ import io.vertx.core.json.Json; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; +import org.folio.inventory.support.JsonHelper; public class Instance { // JSON property names @@ -269,7 +270,7 @@ public JsonObject getJsonForResponse(WebContext context) { putIfNotNull(json, PUBLICATION_KEY, getPublication()); putIfNotNull(json, PUBLICATION_FREQUENCY_KEY, getPublicationFrequency()); putIfNotNull(json, PUBLICATION_RANGE_KEY, getPublicationRange()); - putIfNotNull(json, ELECTRONIC_ACCESS_KEY, getElectronicAccess()); + JsonHelper.putNotNullValues(json, ELECTRONIC_ACCESS_KEY, getElectronicAccess()); putIfNotNull(json, INSTANCE_TYPE_ID_KEY, getInstanceTypeId()); putIfNotNull(json, INSTANCE_FORMAT_IDS_KEY, getInstanceFormatIds()); putIfNotNull(json, PHYSICAL_DESCRIPTIONS_KEY, getPhysicalDescriptions()); diff --git a/src/main/java/org/folio/inventory/domain/items/Item.java b/src/main/java/org/folio/inventory/domain/items/Item.java index af3a0cf92..fe2b65af3 100644 --- a/src/main/java/org/folio/inventory/domain/items/Item.java +++ b/src/main/java/org/folio/inventory/domain/items/Item.java @@ -599,7 +599,8 @@ public Item changeStatus(ItemStatusName newStatus) { .withPurchaseOrderLineIdentifier(purchaseOrderLineIdentifier) .withIsBoundWith(this.isBoundWith) .withTags(tags) - .withCirculationNotes(circulationNotes); + .withCirculationNotes(circulationNotes) + .withLastCheckIn(this.lastCheckIn); } @Override diff --git a/src/main/java/org/folio/inventory/resources/ItemRepresentation.java b/src/main/java/org/folio/inventory/resources/ItemRepresentation.java index fce84872a..2e37cd635 100644 --- a/src/main/java/org/folio/inventory/resources/ItemRepresentation.java +++ b/src/main/java/org/folio/inventory/resources/ItemRepresentation.java @@ -16,6 +16,7 @@ import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; +import org.folio.inventory.support.JsonHelper; class ItemRepresentation { JsonObject toJson(MultipleRecords wrappedItems) { @@ -130,7 +131,7 @@ private JsonObject toJson( includeIfPresent(representation, Item.ITEM_IDENTIFIER_KEY, item.getItemIdentifier()); includeIfPresent(representation,Item.TAGS_KEY, new JsonObject().put(Item.TAG_LIST_KEY, new JsonArray(item.getTags()))); representation.put(Item.YEAR_CAPTION_KEY, item.getYearCaption()); - representation.put(Item.ELECTRONIC_ACCESS_KEY, item.getElectronicAccess()); + JsonHelper.putNotNullValues(representation, Item.ELECTRONIC_ACCESS_KEY, item.getElectronicAccess()); representation.put(Item.STATISTICAL_CODE_IDS_KEY, item.getStatisticalCodeIds()); representation.put(Item.PURCHASE_ORDER_LINE_IDENTIFIER, item.getPurchaseOrderLineIdentifier()); includeReferenceIfPresent(representation, "materialType", @@ -244,7 +245,7 @@ private void includeIfPresent( String propertyName, JsonArray propertyValue) { if (propertyValue != null) { - representation.put( propertyName, propertyValue ); + representation.put(propertyName, propertyValue); } } @@ -257,7 +258,7 @@ private void includeIfPresent( if (from != null) { String value = propertyValue.apply(from); - if(value != null) { + if (value != null) { representation.put(propertyName, value); } } diff --git a/src/main/java/org/folio/inventory/resources/ItemsByHoldingsRecordId.java b/src/main/java/org/folio/inventory/resources/ItemsByHoldingsRecordId.java index 622868ef8..030eb9410 100644 --- a/src/main/java/org/folio/inventory/resources/ItemsByHoldingsRecordId.java +++ b/src/main/java/org/folio/inventory/resources/ItemsByHoldingsRecordId.java @@ -29,7 +29,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -95,7 +94,7 @@ private void getSingleTitleAndMultiTitleItems(RoutingContext routingContext) { webContext, routingContext)) .thenCompose(listOfItems -> maybeAppendSingleTitleItems(listOfItems, holdingsRecordId, webContext, routingContext)) - .thenAccept(combinedListOfItems -> sortAndRespond(combinedListOfItems, + .thenAccept(combinedListOfItems -> respondWithPagedManyItems(combinedListOfItems, webContext, routingContext)); } @@ -177,12 +176,9 @@ private CompletableFuture> maybeAppendSingleTitleItems( return futureListOfItems; } - private void sortAndRespond(List itemList, WebContext webContext, RoutingContext routingContext) { + private void respondWithPagedManyItems(List itemList, WebContext webContext, RoutingContext routingContext) { PagingParameters pagingParameters = getPagingParameters(webContext); - List sortedList = itemList.stream().sorted( - Comparator.comparing(i -> i.getBarcode() == null ? "~" : i.getBarcode())) - .toList(); - MultipleRecords items = new MultipleRecords<>(getPage(sortedList, + MultipleRecords items = new MultipleRecords<>(getPage(itemList, pagingParameters.offset, pagingParameters.limit), itemList.size()); respondWithManyItems(routingContext, webContext, items); } diff --git a/src/main/java/org/folio/inventory/support/JsonHelper.java b/src/main/java/org/folio/inventory/support/JsonHelper.java index efe0a11f6..e46d0d373 100644 --- a/src/main/java/org/folio/inventory/support/JsonHelper.java +++ b/src/main/java/org/folio/inventory/support/JsonHelper.java @@ -1,13 +1,18 @@ package org.folio.inventory.support; -import static org.apache.commons.lang3.StringUtils.isNotBlank; +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Objects; -import io.vertx.core.json.JsonObject; +import static org.apache.commons.lang3.StringUtils.isNotBlank; public class JsonHelper { public static String getNestedProperty( @@ -41,6 +46,27 @@ public JsonObject getJsonFileAsJsonObject(String filePath) throws IOException { return new JsonObject(readFile(is)); } + /** + * Update JsonObject representation with the given property name and only not null values. + * + * @param representation The JsonObject representation to update. + * @param propertyName The name of the property to set. + * @param obj The value to set the property with. + */ + public static void putNotNullValues(JsonObject representation, String propertyName, Object obj) { + if (obj != null && isNotBlank(propertyName)) { + if (obj instanceof Collection collection) { + representation.put(propertyName, new JsonArray(toNotNullList(collection))); + } else if (obj instanceof JsonArray array) { + representation.put(propertyName, toNotNullList(array.getList())); + } else { + JsonObject json = JsonObject.mapFrom(obj); + handleNullNestedFields(json); + representation.put(propertyName, json); + } + } + } + private String readFile(InputStream is) throws IOException { try (BufferedReader br = new BufferedReader(new InputStreamReader(is))) { StringBuilder sb = new StringBuilder(); @@ -53,4 +79,36 @@ private String readFile(InputStream is) throws IOException { return sb.toString(); } } + + private static JsonObject handleNullNestedFields(JsonObject itemObject) { + var keysToRemove = new ArrayList(); + for (var key : itemObject.fieldNames()) { + var value = itemObject.getValue(key); + if (value == null) { + keysToRemove.add(key); + } else if (value instanceof JsonObject object) { + handleNullNestedFields(object); + } + } + + keysToRemove.forEach(itemObject::remove); + return itemObject; + } + + private static List> toNotNullList(Collection collection) { + return collection.stream() + .filter(Objects::nonNull) + .map(item -> { + if (item instanceof Collection iterable) { + return toNotNullList(iterable); + } else if (item instanceof JsonArray array) { + return toNotNullList(array.getList()); + } else { + var jsonItem = JsonObject.mapFrom(item); + handleNullNestedFields(jsonItem); + return jsonItem; + } + }) + .toList(); + } } diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java index a5c55d166..343f5a307 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/actions/CreateInstanceEventHandlerTest.java @@ -41,6 +41,7 @@ import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters; import org.folio.processing.mapping.mapper.reader.Reader; import org.folio.processing.mapping.mapper.reader.record.marc.MarcBibReaderFactory; +import org.folio.processing.value.ListValue; import org.folio.processing.value.MissingValue; import org.folio.processing.value.StringValue; import org.folio.rest.client.SourceStorageRecordsClient; @@ -678,6 +679,44 @@ public class CreateInstanceEventHandlerTest { .withContentType(MAPPING_PROFILE) .withContent(JsonObject.mapFrom(mappingProfile).getMap()))))); + private JobProfile jobProfileWithNatureOfContentTerm = new JobProfile() + .withId(UUID.randomUUID().toString()) + .withName("Create MARC Bibs with NatureOfContentTerm") + .withDataType(JobProfile.DataType.MARC); + + private ActionProfile actionProfileWithNatureOfContentTerm = new ActionProfile() + .withId(UUID.randomUUID().toString()) + .withName("Create preliminary Item with NatureOfContentTerm") + .withAction(ActionProfile.Action.CREATE) + .withFolioRecord(INSTANCE); + + private MappingProfile mappingProfileWithNatureOfContentTerm = new MappingProfile() + .withId(UUID.randomUUID().toString()) + .withName("Prelim item from MARC with NatureOfContentTerm") + .withIncomingRecordType(EntityType.MARC_BIBLIOGRAPHIC) + .withExistingRecordType(EntityType.INSTANCE) + .withMappingDetails(new MappingDetail() + .withMappingFields(Lists.newArrayList( + new MappingRule().withPath("instance.instanceTypeId").withValue("\"instanceTypeIdExpression\"").withEnabled("true"), + new MappingRule().withPath("instance.title").withValue("\"titleExpression\"").withEnabled("true"), + new MappingRule().withPath("instance.natureOfContentTermIds[]").withValue("\"not uuid\"").withEnabled("true").withRepeatableFieldAction(MappingRule.RepeatableFieldAction.EXTEND_EXISTING)))); + + private ProfileSnapshotWrapper profileSnapshotWrapperWithNatureOfContentTerm = new ProfileSnapshotWrapper() + .withId(UUID.randomUUID().toString()) + .withProfileId(jobProfileWithNatureOfContentTerm.getId()) + .withContentType(JOB_PROFILE) + .withContent(jobProfileWithNatureOfContentTerm) + .withChildSnapshotWrappers(Collections.singletonList( + new ProfileSnapshotWrapper() + .withProfileId(actionProfileWithNatureOfContentTerm.getId()) + .withContentType(ACTION_PROFILE) + .withContent(actionProfileWithNatureOfContentTerm) + .withChildSnapshotWrappers(Collections.singletonList( + new ProfileSnapshotWrapper() + .withProfileId(mappingProfileWithNatureOfContentTerm.getId()) + .withContentType(MAPPING_PROFILE) + .withContent(JsonObject.mapFrom(mappingProfileWithNatureOfContentTerm).getMap()))))); + private CreateInstanceEventHandler createInstanceEventHandler; @Before @@ -1092,6 +1131,55 @@ public void shouldNotProcessEventIfRequiredFieldIsEmpty() throws InterruptedExce future.get(5, TimeUnit.MILLISECONDS); } + @Test(expected = Exception.class) + public void shouldNotProcessEventIfNatureContentFieldIsNotUUID() throws InterruptedException, ExecutionException, TimeoutException { + Reader fakeReader = Mockito.mock(Reader.class); + + String instanceTypeId = "fe19bae4-da28-472b-be90-d442e2428ead"; + String recordId = "567859ad-505a-400d-a699-0028a1fdbf84"; + String instanceId = "4d4545df-b5ba-4031-a031-70b1c1b2fc5d"; + String title = "titleValue"; + RecordToEntity recordToInstance = RecordToEntity.builder().recordId(recordId).entityId(instanceId) + .build(); + + when(fakeReader.read(any(MappingRule.class))).thenReturn(StringValue.of(instanceTypeId), StringValue.of(title), ListValue.of(Lists.newArrayList("not uuid"))); + + when(fakeReaderFactory.createReader()).thenReturn(fakeReader); + + when(storage.getInstanceCollection(any())).thenReturn(instanceRecordCollection); + + when(instanceIdStorageService.store(any(), any(), any())).thenReturn(Future.succeededFuture(recordToInstance)); + + MappingManager.registerReaderFactory(fakeReaderFactory); + MappingManager.registerWriterFactory(new InstanceWriterFactory()); + + HashMap context = new HashMap<>(); + Record record = new Record().withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT)); + record.setId(recordId); + + context.put(MARC_BIBLIOGRAPHIC.value(), Json.encode(record)); + + Buffer buffer = BufferImpl.buffer("{\"parsedRecord\":{" + + "\"id\":\"990fad8b-64ec-4de4-978c-9f8bbed4c6d3\"," + + "\"content\":\"{\\\"leader\\\":\\\"00574nam 22001211a 4500\\\",\\\"fields\\\":[{\\\"035\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"(in001)ybp7406411\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"245\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"titleValue\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"336\\\":{\\\"subfields\\\":[{\\\"b\\\":\\\"b6698d38-149f-11ec-82a8-0242ac130003\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"780\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"Houston oil directory\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"785\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"SAIS review of international affairs\\\"},{\\\"x\\\":\\\"1945-4724\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"500\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Adaptation of Xi xiang ji by Wang Shifu.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"520\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"999\\\":{\\\"subfields\\\":[{\\\"i\\\":\\\"4d4545df-b5ba-4031-a031-70b1c1b2fc5d\\\"}],\\\"ind1\\\":\\\"f\\\",\\\"ind2\\\":\\\"f\\\"}}]}\"" + + "}}"); + HttpResponse resp = buildHttpResponseWithBuffer(buffer); + when(sourceStorageClient.postSourceStorageRecords(any())).thenReturn(Future.succeededFuture(resp)); + + DataImportEventPayload dataImportEventPayload = new DataImportEventPayload() + .withEventType(DI_INVENTORY_INSTANCE_CREATED.value()) + .withContext(context) + .withCurrentNode(profileSnapshotWrapperWithNatureOfContentTerm.getChildSnapshotWrappers().get(0)) + .withTenant(TENANT_ID) + .withOkapiUrl(mockServer.baseUrl()) + .withToken(TOKEN) + .withJobExecutionId(UUID.randomUUID().toString()) + .withOkapiUrl(mockServer.baseUrl()); + + CompletableFuture future = createInstanceEventHandler.handle(dataImportEventPayload); + future.get(10, TimeUnit.SECONDS); + } + @Test(expected = Exception.class) public void shouldNotProcessEventIfRecordContains999field() throws InterruptedException, ExecutionException, TimeoutException { var recordId = UUID.randomUUID().toString(); diff --git a/src/test/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandlerTest.java b/src/test/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandlerTest.java index a21d75899..d36460f71 100644 --- a/src/test/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandlerTest.java +++ b/src/test/java/org/folio/inventory/dataimport/handlers/actions/ReplaceInstanceEventHandlerTest.java @@ -41,6 +41,7 @@ import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters; import org.folio.processing.mapping.mapper.reader.Reader; import org.folio.processing.mapping.mapper.reader.record.marc.MarcBibReaderFactory; +import org.folio.processing.value.ListValue; import org.folio.processing.value.MissingValue; import org.folio.processing.value.StringValue; import org.folio.rest.client.SourceStorageRecordsClient; @@ -198,6 +199,44 @@ public class ReplaceInstanceEventHandlerTest { .withContentType(MAPPING_PROFILE) .withContent(JsonObject.mapFrom(mappingProfile).getMap()))))); + private JobProfile jobProfileWithNatureOfContentTerm = new JobProfile() + .withId(UUID.randomUUID().toString()) + .withName("Create MARC Bibs with NatureOfContentTerm") + .withDataType(JobProfile.DataType.MARC); + + private ActionProfile actionProfileWithNatureOfContentTerm = new ActionProfile() + .withId(UUID.randomUUID().toString()) + .withName("Replace preliminary Item with NatureOfContentTerm") + .withAction(ActionProfile.Action.UPDATE) + .withFolioRecord(INSTANCE); + + private MappingProfile mappingProfileWithNatureOfContentTerm = new MappingProfile() + .withId(UUID.randomUUID().toString()) + .withName("Prelim item from MARC with NatureOfContentTerm") + .withIncomingRecordType(EntityType.MARC_BIBLIOGRAPHIC) + .withExistingRecordType(EntityType.INSTANCE) + .withMappingDetails(new MappingDetail() + .withMappingFields(Lists.newArrayList( + new MappingRule().withPath("instance.instanceTypeId").withValue("\"instanceTypeIdExpression\"").withEnabled("true"), + new MappingRule().withPath("instance.title").withValue("\"titleExpression\"").withEnabled("true"), + new MappingRule().withPath("instance.natureOfContentTermIds[]").withValue("\"not uuid\"").withEnabled("true").withRepeatableFieldAction(MappingRule.RepeatableFieldAction.EXTEND_EXISTING)))); + + private ProfileSnapshotWrapper profileSnapshotWrapperWithNatureOfContentTerm = new ProfileSnapshotWrapper() + .withId(UUID.randomUUID().toString()) + .withProfileId(jobProfileWithNatureOfContentTerm.getId()) + .withContentType(JOB_PROFILE) + .withContent(jobProfileWithNatureOfContentTerm) + .withChildSnapshotWrappers(Collections.singletonList( + new ProfileSnapshotWrapper() + .withProfileId(actionProfileWithNatureOfContentTerm.getId()) + .withContentType(ACTION_PROFILE) + .withContent(actionProfileWithNatureOfContentTerm) + .withChildSnapshotWrappers(Collections.singletonList( + new ProfileSnapshotWrapper() + .withProfileId(mappingProfileWithNatureOfContentTerm.getId()) + .withContentType(MAPPING_PROFILE) + .withContent(JsonObject.mapFrom(mappingProfileWithNatureOfContentTerm).getMap()))))); + private ReplaceInstanceEventHandler replaceInstanceEventHandler; private PrecedingSucceedingTitlesHelper precedingSucceedingTitlesHelper; @@ -824,6 +863,56 @@ public void shouldNotProcessEventIfRequiredFieldIsEmpty() throws InterruptedExce future.get(5, TimeUnit.MILLISECONDS); } + @Test(expected = ExecutionException.class) + public void shouldNotProcessEventIfNatureContentFieldIsNotUUID() throws InterruptedException, ExecutionException, TimeoutException { + Reader fakeReader = Mockito.mock(Reader.class); + + String instanceTypeId = UUID.randomUUID().toString(); + String title = "titleValue"; + + when(fakeReader.read(any(MappingRule.class))).thenReturn(StringValue.of(instanceTypeId), StringValue.of(title), ListValue.of(Lists.newArrayList("not uuid"))); + + when(fakeReaderFactory.createReader()).thenReturn(fakeReader); + + when(storage.getInstanceCollection(any())).thenReturn(instanceRecordCollection); + + MappingManager.registerReaderFactory(fakeReaderFactory); + MappingManager.registerWriterFactory(new InstanceWriterFactory()); + + HashMap context = new HashMap<>(); + Record record = new Record().withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT)); + context.put(MARC_BIBLIOGRAPHIC.value(), Json.encode(record)); + context.put(INSTANCE.value(), new JsonObject() + .put("id", instanceId) + .put("hrid", UUID.randomUUID().toString()) + .put("source", MARC_INSTANCE_SOURCE) + .put("_version", INSTANCE_VERSION) + .put("discoverySuppress", false) + .encode()); + + mockInstance(MARC_INSTANCE_SOURCE); + + Buffer buffer = BufferImpl.buffer("{\"parsedRecord\":{" + + "\"id\":\"990fad8b-64ec-4de4-978c-9f8bbed4c6d3\"," + + "\"content\":\"{\\\"leader\\\":\\\"00574nam 22001211a 4500\\\",\\\"fields\\\":[{\\\"035\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"(in001)ybp7406411\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"245\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"titleValue\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"336\\\":{\\\"subfields\\\":[{\\\"b\\\":\\\"b6698d38-149f-11ec-82a8-0242ac130003\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"780\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"Houston oil directory\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"785\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"SAIS review of international affairs\\\"},{\\\"x\\\":\\\"1945-4724\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"500\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Adaptation of Xi xiang ji by Wang Shifu.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"520\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"999\\\":{\\\"subfields\\\":[{\\\"i\\\":\\\"4d4545df-b5ba-4031-a031-70b1c1b2fc5d\\\"}],\\\"ind1\\\":\\\"f\\\",\\\"ind2\\\":\\\"f\\\"}}]}\"" + + "}}"); + HttpResponse respForPass = buildHttpResponseWithBuffer(buffer, HttpStatus.SC_OK); + when(sourceStorageClient.putSourceStorageRecordsGenerationById(any(), any())).thenReturn(Future.succeededFuture(respForPass)); + + DataImportEventPayload dataImportEventPayload = new DataImportEventPayload() + .withEventType(DI_INVENTORY_INSTANCE_CREATED.value()) + .withContext(context) + .withCurrentNode(profileSnapshotWrapperWithNatureOfContentTerm.getChildSnapshotWrappers().get(0)) + .withTenant(TENANT_ID) + .withOkapiUrl(mockServer.baseUrl()) + .withToken(TOKEN) + .withJobExecutionId(UUID.randomUUID().toString()); + + CompletableFuture future = replaceInstanceEventHandler.handle(dataImportEventPayload); + future.get(10, TimeUnit.SECONDS); + } + + @Test public void shouldReturnFailedFutureIfCurrentActionProfileHasNoMappingProfile() { HashMap context = new HashMap<>(); diff --git a/src/test/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtilTest.java b/src/test/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtilTest.java index f0357f07c..5f3786f51 100644 --- a/src/test/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtilTest.java +++ b/src/test/java/org/folio/inventory/dataimport/util/AdditionalFieldsUtilTest.java @@ -3,6 +3,7 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.folio.inventory.TestUtil; @@ -17,7 +18,9 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.BlockJUnit4ClassRunner; +import org.marc4j.MarcException; +import java.io.File; import java.io.IOException; import java.time.Instant; import java.time.ZoneId; @@ -27,7 +30,17 @@ import java.util.UUID; import java.util.stream.IntStream; -import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.*; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_001; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_005; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.TAG_999; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.addControlledFieldToMarcRecord; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.addDataFieldToMarcRecord; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.addFieldToMarcRecord; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.dateTime005Formatter; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.getCacheStats; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.getValueFromControlledField; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.isFieldExist; +import static org.folio.inventory.dataimport.util.AdditionalFieldsUtil.removeField; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.*; @@ -35,6 +48,9 @@ public class AdditionalFieldsUtilTest { private static final String PARSED_MARC_RECORD_PATH = "src/test/resources/marc/parsedMarcRecord.json"; + private static final String PARSED_RECORD = "src/test/resources/marc/parsedRecord.json"; + private static final String REORDERED_PARSED_RECORD = "src/test/resources/marc/reorderedParsedRecord.json"; + private static final String REORDERING_RESULT_RECORD = "src/test/resources/marc/reorderingResultRecord.json"; @Test public void shouldAddInstanceIdSubfield() throws IOException { @@ -47,8 +63,8 @@ public void shouldAddInstanceIdSubfield() throws IOException { String leader = new JsonObject(parsedRecordContent).getString("leader"); Record record = new Record().withId(recordId).withParsedRecord(parsedRecord); // when - boolean addedSourceRecordId = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 's', recordId); - boolean addedInstanceId = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean addedSourceRecordId = addFieldToMarcRecord(record, TAG_999, 's', recordId); + boolean addedInstanceId = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertTrue(addedSourceRecordId); Assert.assertTrue(addedInstanceId); @@ -60,8 +76,8 @@ public void shouldAddInstanceIdSubfield() throws IOException { int totalFieldsCount = 0; for (int i = fields.size(); i-- > 0; ) { JsonObject targetField = fields.getJsonObject(i); - if (targetField.containsKey(AdditionalFieldsUtil.TAG_999)) { - JsonArray subfields = targetField.getJsonObject(AdditionalFieldsUtil.TAG_999).getJsonArray("subfields"); + if (targetField.containsKey(TAG_999)) { + JsonArray subfields = targetField.getJsonObject(TAG_999).getJsonArray("subfields"); for (int j = subfields.size(); j-- > 0; ) { JsonObject targetSubfield = subfields.getJsonObject(j); if (targetSubfield.containsKey("i")) { @@ -85,7 +101,7 @@ public void shouldNotAddInstanceIdSubfieldIfNoParsedRecordContent() { Record record = new Record(); String instanceId = UUID.randomUUID().toString(); // when - boolean added = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean added = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertFalse(added); Assert.assertNull(record.getParsedRecord()); @@ -99,7 +115,7 @@ public void shouldNotAddInstanceIdSubfieldIfNoFieldsInParsedRecordContent() { record.setParsedRecord(new ParsedRecord().withContent(content)); String instanceId = UUID.randomUUID().toString(); // when - boolean added = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean added = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertFalse(added); Assert.assertNotNull(record.getParsedRecord()); @@ -115,7 +131,7 @@ public void shouldNotAddInstanceIdSubfieldIfCanNotConvertParsedContentToJsonObje record.setParsedRecord(new ParsedRecord().withContent(content)); String instanceId = UUID.randomUUID().toString(); // when - boolean added = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean added = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertFalse(added); Assert.assertNotNull(record.getParsedRecord()); @@ -131,7 +147,7 @@ public void shouldNotAddInstanceIdSubfieldIfContentHasNoFields() { record.setParsedRecord(new ParsedRecord().withContent(content)); String instanceId = UUID.randomUUID().toString(); // when - boolean added = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean added = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertFalse(added); Assert.assertNotNull(record.getParsedRecord()); @@ -145,7 +161,7 @@ public void shouldNotAddInstanceIdSubfieldIfContentIsNull() { record.setParsedRecord(new ParsedRecord().withContent(null)); String instanceId = UUID.randomUUID().toString(); // when - boolean added = AdditionalFieldsUtil.addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId); + boolean added = addFieldToMarcRecord(record, TAG_999, 'i', instanceId); // then Assert.assertFalse(added); Assert.assertNotNull(record.getParsedRecord()); @@ -160,7 +176,7 @@ public void shouldRemoveField() throws IOException { String leader = new JsonObject(parsedRecordContent).getString("leader"); parsedRecord.setContent(parsedRecordContent); Record record = new Record().withId(recordId).withParsedRecord(parsedRecord); - boolean deleted = AdditionalFieldsUtil.removeField(record, "001"); + boolean deleted = removeField(record, "001"); Assert.assertTrue(deleted); JsonObject content = new JsonObject(parsedRecord.getContent().toString()); JsonArray fields = content.getJsonArray("fields"); @@ -178,7 +194,7 @@ public void shouldNotAddControlledFieldToMarcRecord() throws IOException { String parsedRecordContent = TestUtil.readFileFromPath(PARSED_MARC_RECORD_PATH); ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedRecordContent); Record record = new Record().withId(recordId).withParsedRecord(parsedRecord); - boolean added = AdditionalFieldsUtil.addControlledFieldToMarcRecord(record, "002", "", null); + boolean added = addControlledFieldToMarcRecord(record, "002", "", null); Assert.assertFalse(added); } @@ -189,7 +205,7 @@ public void shouldAddControlledFieldToMarcRecord() throws IOException { ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedRecordContent); String leader = new JsonObject(parsedRecordContent).getString("leader"); Record record = new Record().withId(recordId).withParsedRecord(parsedRecord); - boolean added = AdditionalFieldsUtil.addControlledFieldToMarcRecord( + boolean added = addControlledFieldToMarcRecord( record, "002", "test", AdditionalFieldsUtil::addControlledFieldToMarcRecord); Assert.assertTrue(added); JsonObject content = new JsonObject(parsedRecord.getContent().toString()); @@ -209,7 +225,7 @@ public void shouldReplaceControlledFieldInMarcRecord() throws IOException { ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedRecordContent); String leader = new JsonObject(parsedRecordContent).getString("leader"); Record record = new Record().withId(recordId).withParsedRecord(parsedRecord); - boolean added = AdditionalFieldsUtil.addControlledFieldToMarcRecord( + boolean added = addControlledFieldToMarcRecord( record, "003", "test", AdditionalFieldsUtil::replaceOrAddControlledFieldInMarcRecord); Assert.assertTrue(added); JsonObject content = new JsonObject(parsedRecord.getContent().toString()); @@ -248,7 +264,7 @@ public void shouldAddFieldToMarcRecordInNumericalOrder() throws IOException { String leader = new JsonObject(parsedRecordContent).getString("leader"); Record record = new Record().withId(UUID.randomUUID().toString()).withParsedRecord(parsedRecord); // when - boolean added = AdditionalFieldsUtil.addDataFieldToMarcRecord(record, "035", ' ', ' ', 'a', instanceHrId); + boolean added = addDataFieldToMarcRecord(record, "035", ' ', ' ', 'a', instanceHrId); // then Assert.assertTrue(added); JsonObject content = new JsonObject(parsedRecord.getContent().toString()); @@ -278,7 +294,7 @@ public void shouldNotSortExistingFieldsWhenAddFieldToToMarcRecord() { ParsedRecord parsedRecord = new ParsedRecord().withContent(parsedContent); Record record = new Record().withId(UUID.randomUUID().toString()).withParsedRecord(parsedRecord); // when - boolean added = AdditionalFieldsUtil.addDataFieldToMarcRecord(record, "999", 'f', 'f', 'i', instanceId); + boolean added = addDataFieldToMarcRecord(record, "999", 'f', 'f', 'i', instanceId); // then Assert.assertTrue(added); Assert.assertEquals(expectedParsedContent, parsedRecord.getContent()); @@ -387,7 +403,7 @@ public void shouldNotUpdate005Field() { AdditionalFieldsUtil.updateLatestTransactionDate(record, new MappingParameters().withMarcFieldProtectionSettings(List.of(new MarcFieldProtectionSetting().withField("*").withData("*")))); - String actualDate = AdditionalFieldsUtil.getValueFromControlledField(record, TAG_005); + String actualDate = getValueFromControlledField(record, TAG_005); assertNotNull(actualDate); assertEquals("20141107001016.0", actualDate); } @@ -405,7 +421,7 @@ public void shouldUpdate005Field() { AdditionalFieldsUtil.updateLatestTransactionDate(record, new MappingParameters()); - String actualDate = AdditionalFieldsUtil.getValueFromControlledField(record, TAG_005); + String actualDate = getValueFromControlledField(record, TAG_005); assertNotNull(actualDate); assertEquals(expectedDate.substring(0, 10), actualDate.substring(0, 10)); } @@ -539,7 +555,7 @@ public void caching() throws IOException { Assert.assertEquals(2, cacheStats.missCount()); Assert.assertEquals(2, cacheStats.loadCount()); // add field to marc record - Assert.assertTrue(addFieldToMarcRecord(record, AdditionalFieldsUtil.TAG_999, 'i', instanceId)); + Assert.assertTrue(addFieldToMarcRecord(record, TAG_999, 'i', instanceId)); cacheStats = getCacheStats().minus(initialCacheStats); Assert.assertEquals(9, cacheStats.requestCount()); Assert.assertEquals(7, cacheStats.hitCount()); @@ -636,14 +652,14 @@ public void shouldRemove035() { @Test public void shouldReorderMarcRecordFields() throws IOException, MarcException { - var systemReorderedRecordContent = readFileFromPath(PARSED_RECORD); - var userOrderRecordContent = readFileFromPath(REORDERED_PARSED_RECORD); - var expectedOrderRecord = readFileFromPath(REORDERING_RESULT_RECORD); + var reorderedRecordContent = readFileFromPath(PARSED_RECORD); + var sourceRecordContent = readFileFromPath(REORDERED_PARSED_RECORD); + var reorderingResultRecord = readFileFromPath(REORDERING_RESULT_RECORD); - var actualOrderRecord = AdditionalFieldsUtil.reorderMarcRecordFields(userOrderRecordContent, systemReorderedRecordContent); + var resultContent = AdditionalFieldsUtil.reorderMarcRecordFields(sourceRecordContent, reorderedRecordContent); - assertNotNull(actualOrderRecord); - assertEquals(formatContent(expectedOrderRecord), formatContent(actualOrderRecord)); + assertNotNull(resultContent); + assertEquals(formatContent(resultContent), formatContent(reorderingResultRecord)); } private static String readFileFromPath(String path) throws IOException { diff --git a/src/test/java/org/folio/inventory/dataimport/util/ValidationUtilTest.java b/src/test/java/org/folio/inventory/dataimport/util/ValidationUtilTest.java new file mode 100644 index 000000000..69eba9a19 --- /dev/null +++ b/src/test/java/org/folio/inventory/dataimport/util/ValidationUtilTest.java @@ -0,0 +1,34 @@ +package org.folio.inventory.dataimport.util; + +import org.folio.inventory.domain.instances.Instance; +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.BlockJUnit4ClassRunner; + +import java.util.Arrays; +import java.util.List; +import java.util.UUID; + +@RunWith(BlockJUnit4ClassRunner.class) +public class ValidationUtilTest { + + @Test + public void shouldHaveNoErrorIfNatureAreAsUUID() { + Instance instance = new Instance(UUID.randomUUID().toString(), "1", "001", "MARC", "Title", UUID.randomUUID().toString()); + instance.setNatureOfContentTermIds(Arrays.asList(UUID.randomUUID().toString(), UUID.randomUUID().toString())); + List errors = ValidationUtil.validateUUIDs(instance); + Assert.assertEquals(0, errors.size()); + } + + @Test + public void shouldHaveSeveralErrorsIfSomeNatureAreNotAsUUID() { + Instance instance = new Instance(UUID.randomUUID().toString(), "1", "001", "MARC", "Title", UUID.randomUUID().toString()); + instance.setNatureOfContentTermIds(Arrays.asList(UUID.randomUUID().toString(), "not uuid value", UUID.randomUUID().toString(), UUID.randomUUID().toString(), "second not UUID value")); + List errors = ValidationUtil.validateUUIDs(instance); + Assert.assertEquals(2, errors.size()); + Assert.assertEquals("Value 'not uuid value' is not a UUID for natureOfContentTermIds field", errors.get(0)); + Assert.assertEquals("Value 'second not UUID value' is not a UUID for natureOfContentTermIds field", errors.get(1)); + + } +} diff --git a/src/test/java/org/folio/inventory/resources/ItemRepresentationTest.java b/src/test/java/org/folio/inventory/resources/ItemRepresentationTest.java index 36414e0cf..31e5f8b82 100644 --- a/src/test/java/org/folio/inventory/resources/ItemRepresentationTest.java +++ b/src/test/java/org/folio/inventory/resources/ItemRepresentationTest.java @@ -1,17 +1,19 @@ package org.folio.inventory.resources; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.is; - -import java.util.UUID; - +import io.vertx.core.json.JsonArray; +import io.vertx.core.json.JsonObject; import org.folio.inventory.domain.items.Item; import org.folio.inventory.domain.items.ItemStatusName; import org.folio.inventory.domain.items.Status; +import org.folio.inventory.domain.sharedproperties.ElectronicAccess; import org.junit.Test; -import io.vertx.core.json.JsonArray; -import io.vertx.core.json.JsonObject; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; public class ItemRepresentationTest { @Test @@ -23,4 +25,29 @@ public void jsonContainsVersion() { .toJson(item, null, instance, null, null, null, null, null, null); assertThat(json.getString("_version"), is("123")); } + + @Test + public void jsonWithoutNullValuesOnly() { + var testValue = "https://test.com"; + var electronicAccessKey = "electronicAccess"; + var item = new Item(UUID.randomUUID().toString(), null, null, + new Status(ItemStatusName.AVAILABLE), null, null, null); + var electronicAccess = new ElectronicAccess(testValue, "", null, null, null); + item.withElectronicAccess(List.of(electronicAccess)); + var json = new ItemRepresentation() + .toJson(item, null, new JsonObject().put("contributors", new JsonArray()), null, null, null, null, null, null); + var electronicAccessObject = json.getJsonArray(electronicAccessKey); + + var nullableList = new ArrayList(); + nullableList.add(null); + item.withElectronicAccess(nullableList); + var nullElectronicAccessJson = new ItemRepresentation() + .toJson(item, null, new JsonObject().put("contributors", new JsonArray()), null, null, null, null, null, null); + var emptyElectronicAccessObject = nullElectronicAccessJson.getJsonArray(electronicAccessKey); + + assertThat(electronicAccessObject.size(), is(1)); + assertThat(electronicAccessObject.getJsonObject(0).fieldNames().size(), is(2)); + assertThat(electronicAccessObject.getJsonObject(0).getValue("uri"), is(testValue)); + assertThat(emptyElectronicAccessObject, is(new JsonArray())); + } } diff --git a/src/test/java/org/folio/inventory/support/JsonHelperTest.java b/src/test/java/org/folio/inventory/support/JsonHelperTest.java index 690a2e340..60f9510b5 100644 --- a/src/test/java/org/folio/inventory/support/JsonHelperTest.java +++ b/src/test/java/org/folio/inventory/support/JsonHelperTest.java @@ -1,17 +1,23 @@ package org.folio.inventory.support; -import static org.folio.inventory.support.JsonHelper.includeIfPresent; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyZeroInteractions; - +import io.vertx.core.json.JsonArray; import io.vertx.core.json.JsonObject; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Spy; import org.mockito.junit.MockitoJUnitRunner; +import java.util.ArrayList; + +import static org.folio.inventory.support.JsonHelper.includeIfPresent; +import static org.folio.inventory.support.JsonHelper.putNotNullValues; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; +import static org.mockito.internal.verification.VerificationModeFactory.times; + @RunWith(MockitoJUnitRunner.class) public class JsonHelperTest { @Spy @@ -43,4 +49,44 @@ public void shouldNotIncludeIfValueIsNull() { includeIfPresent(representation, "key", null); verifyZeroInteractions(representation); } + + @Test + public void shouldNotIncludeOnlyNullValues() { + var notNullString = "notNull"; + var key = "key"; + var rootKey = "root"; + var arrayKey = "array"; + var jsonArrayKey = "jsonArray"; + var arrayListKey = "arrayList"; + var value = new JsonObject(); + var nestedValue = new JsonObject(); + var list = new ArrayList(); + var arrayList = new ArrayList(); + var jsonArray = new JsonArray(); + + nestedValue.put(notNullString, notNullString); + nestedValue.put("null", null); + nestedValue.put("empty", ""); + value.put(key, nestedValue); + list.add(nestedValue); + list.add(null); + arrayList.add(JsonArray.of(nestedValue)); + jsonArray.add(nestedValue); + putNotNullValues(representation, rootKey, value); + putNotNullValues(representation, arrayKey, list); + putNotNullValues(representation, arrayListKey, arrayList); + putNotNullValues(representation, jsonArrayKey, jsonArray); + + var objResult = representation.getJsonObject(rootKey).getJsonObject(key); + var listResult = representation.getJsonArray(arrayKey); + var arrayListResult = representation.getJsonArray(arrayListKey).getJsonArray(0).getJsonObject(0); + var jsonArrayResult = representation.getJsonArray(jsonArrayKey).getJsonObject(0); + assertThat(objResult.size(), is(2)); + assertThat(objResult.getValue(notNullString), is(notNullString)); + assertThat(arrayListResult.size(), is(2)); + assertThat(arrayListResult.getValue(notNullString), is(notNullString)); + assertThat(jsonArrayResult.size(), is(2)); + assertThat(jsonArrayResult.getValue(notNullString), is(notNullString)); + assertThat(listResult.getList().size(), is(1)); + } } diff --git a/src/test/resources/marc/parsedRecord.json b/src/test/resources/marc/parsedRecord.json new file mode 100644 index 000000000..4c6d25e55 --- /dev/null +++ b/src/test/resources/marc/parsedRecord.json @@ -0,0 +1,335 @@ +{ + "leader":"01314nam 22003851a 4500", + "fields":[ + { + "001":"ybp7406411" + }, + { + "005":"20120404100627.6" + }, + { + "003":"NhCcYBP" + }, + { + "006":"m||||||||d|||||||" + }, + { + "007":"cr||n|||||||||" + }, + { + "008":"120329s2011 sz a ob 001 0 eng d" + }, + { + "020":{ + "subfields":[ + { + "a":"2940447241 (electronic bk.)" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "020":{ + "subfields":[ + { + "a":"9782940447244 (electronic bk.)" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "040":{ + "subfields":[ + { + "a":"NhCcYBP" + }, + { + "c":"NhCcYBP" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "050":{ + "subfields":[ + { + "a":"Z246" + }, + { + "b":".A43 2011" + } + ], + "ind1":" ", + "ind2":"4" + } + }, + { + "082":{ + "subfields":[ + { + "a":"686.22" + }, + { + "2":"22" + } + ], + "ind1":"0", + "ind2":"4" + } + }, + { + "100":{ + "subfields":[ + { + "a":"Ambrose, Gavin." + } + ], + "ind1":"1", + "ind2":" " + } + }, + { + "245":{ + "subfields":[ + { + "a":"The fundamentals of typography" + }, + { + "h":"[electronic resource] /" + }, + { + "c":"Gavin Ambrose, Paul Harris." + } + ], + "ind1":"1", + "ind2":"4" + } + }, + { + "250":{ + "subfields":[ + { + "a":"2nd ed." + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "260":{ + "subfields":[ + { + "a":"Lausanne ;" + }, + { + "a":"Worthing :" + }, + { + "b":"AVA Academia," + }, + { + "c":"2011." + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "300":{ + "subfields":[ + { + "a":"1 online resource." + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "490":{ + "subfields":[ + { + "a":"AVA Academia series" + } + ], + "ind1":"1", + "ind2":" " + } + }, + { + "500":{ + "subfields":[ + { + "a":"Previous ed.: 2006." + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "504":{ + "subfields":[ + { + "a":"Includes bibliographical references (p. [200]) and index." + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "505":{ + "subfields":[ + { + "a":"Type and language -- A few basics -- Letterforms -- Words and paragraphs -- Using type." + } + ], + "ind1":"0", + "ind2":" " + } + }, + { + "650":{ + "subfields":[ + { + "a":"Graphic design (Typography)" + } + ], + "ind1":" ", + "ind2":"0" + } + }, + { + "650":{ + "subfields":[ + { + "a":"Printing." + } + ], + "ind1":" ", + "ind2":"0" + } + }, + { + "700":{ + "subfields":[ + { + "a":"Harris, Paul," + }, + { + "d":"1971-" + } + ], + "ind1":"1", + "ind2":" " + } + }, + { + "710":{ + "subfields":[ + { + "a":"EBSCOhost" + } + ], + "ind1":"2", + "ind2":" " + } + }, + { + "776":{ + "subfields":[ + { + "c":"Original" + }, + { + "z":"9782940411764" + }, + { + "z":"294041176X" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "830":{ + "subfields":[ + { + "a":"AVA academia." + } + ], + "ind1":" ", + "ind2":"0" + } + }, + { + "856":{ + "subfields":[ + { + "u":"http://search.ebscohost.com/login.aspx?direct=true&scope=site&db=nlebk&db=nlabk&AN=430135" + } + ], + "ind1":"4", + "ind2":"0" + } + }, + { + "935":{ + "subfields":[ + { + "a":".o13465259" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "980":{ + "subfields":[ + { + "a":"130307" + }, + { + "b":"7107" + }, + { + "e":"7107" + }, + { + "f":"243965" + }, + { + "g":"1" + } + ], + "ind1":" ", + "ind2":" " + } + }, + { + "981":{ + "subfields":[ + { + "b":"OM" + }, + { + "c":"nlnet" + } + ], + "ind1":" ", + "ind2":" " + } + } + ] +}