From f3f972f411393b55ed8572503bbfcfa99add6e3a Mon Sep 17 00:00:00 2001 From: Volodymyr_Rohach Date: Tue, 2 Jan 2024 14:51:08 +0200 Subject: [PATCH] MODSOURMAN-1063: Implementation finished. --- NEWS.md | 1 + .../java/org/folio/dao/JournalRecordDao.java | 6 +- .../org/folio/dao/JournalRecordDaoImpl.java | 105 +-- .../folio/rest/impl/MetadataProviderImpl.java | 2 +- .../folio/services/JournalRecordService.java | 6 +- .../services/JournalRecordServiceImpl.java | 6 +- .../create_get_job_log_entries_function.sql | 650 +++++++++++++++--- ramls/metadata-provider.raml | 7 +- ramls/tmpSchemas/recordProcessingLogDto.json | 130 ++++ .../recordProcessingLogDtoCollection.json | 27 + 10 files changed, 802 insertions(+), 138 deletions(-) create mode 100644 ramls/tmpSchemas/recordProcessingLogDto.json create mode 100644 ramls/tmpSchemas/recordProcessingLogDtoCollection.json diff --git a/NEWS.md b/NEWS.md index eb21a1e05..c6fedeed3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -4,6 +4,7 @@ * [MODSOURMAN-1021](https://issues.folio.org/browse/MODSOURMAN-1021) Provide endpoint for getting parsed content for DI log * [MODSOURMAN-1022](https://issues.folio.org/browse/MODSOURMAN-1022) Remove step of initial saving of incoming records to SRS * [MODSOURMAN-1070](https://issues.folio.org/browse/MODSOURMAN-1070) Fill in Journal Records for created MARC when INSTANCE_CREATED event received +* [MODSOURMAN-1063](https://issues.folio.org/browse/MODSOURMAN-1063) Update RecordProcessingLogDto to contain incoming record id ## 2023-10-13 v3.7.0 * [MODSOURMAN-1045](https://issues.folio.org/browse/MODSOURMAN-1045) Allow create action with non-matches for instance without match profile diff --git a/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDao.java b/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDao.java index b4bd5b3c4..072bb74a0 100644 --- a/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDao.java +++ b/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDao.java @@ -4,9 +4,9 @@ import io.vertx.sqlclient.Row; import io.vertx.sqlclient.RowSet; import org.folio.rest.jaxrs.model.JobExecutionSummaryDto; -import org.folio.rest.jaxrs.model.JobLogEntryDtoCollection; import org.folio.rest.jaxrs.model.JournalRecord; import org.folio.rest.jaxrs.model.RecordProcessingLogDto; +import org.folio.rest.jaxrs.model.RecordProcessingLogDtoCollection; import java.util.List; import java.util.Optional; @@ -55,7 +55,7 @@ public interface JournalRecordDao { Future deleteByJobExecutionId(String jobExecutionId, String tenantId); /** - * Searches for JobLogEntryDto entities by jobExecutionId and sorts them using specified sort criteria and direction + * Searches for RecordProcessingLogDtoCollection by jobExecutionId and sorts them using specified sort criteria and direction * * @param jobExecutionId job execution id * @param sortBy sorting criteria @@ -67,7 +67,7 @@ public interface JournalRecordDao { * @param tenantId tenantId * @return future with JobLogEntryDto collection */ - Future getJobLogEntryDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId); + Future getRecordProcessingLogDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId); /** * Searches for RecordProcessingLogDto entities by jobExecutionId and recordId diff --git a/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDaoImpl.java b/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDaoImpl.java index d9110d669..c0406a439 100644 --- a/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDaoImpl.java +++ b/mod-source-record-manager-server/src/main/java/org/folio/dao/JournalRecordDaoImpl.java @@ -6,14 +6,12 @@ import io.vertx.sqlclient.RowSet; import io.vertx.sqlclient.SqlResult; import io.vertx.sqlclient.Tuple; -import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.folio.dao.util.JournalRecordsColumns; import org.folio.dao.util.PostgresClientFactory; import org.folio.rest.jaxrs.model.EntityProcessingSummary; import org.folio.rest.jaxrs.model.JobExecutionSummaryDto; -import org.folio.rest.jaxrs.model.JobLogEntryDto; -import org.folio.rest.jaxrs.model.JobLogEntryDtoCollection; import org.folio.rest.jaxrs.model.JournalRecord; import org.folio.rest.jaxrs.model.JournalRecord.ActionStatus; import org.folio.rest.jaxrs.model.JournalRecord.ActionType; @@ -22,6 +20,7 @@ import org.folio.rest.jaxrs.model.ProcessedHoldingsInfo; import org.folio.rest.jaxrs.model.ProcessedItemInfo; import org.folio.rest.jaxrs.model.RecordProcessingLogDto; +import org.folio.rest.jaxrs.model.RecordProcessingLogDtoCollection; import org.folio.rest.jaxrs.model.RelatedInvoiceLineInfo; import org.folio.rest.jaxrs.model.RelatedPoLineInfo; import org.springframework.beans.factory.annotation.Autowired; @@ -78,13 +77,17 @@ import static org.folio.dao.util.JournalRecordsColumns.INVOICE_LINE_ENTITY_ERROR; import static org.folio.dao.util.JournalRecordsColumns.INVOICE_LINE_ENTITY_HRID; import static org.folio.dao.util.JournalRecordsColumns.INVOICE_LINE_ENTITY_ID; -import static org.folio.dao.util.JournalRecordsColumns.INVOICE_LINE_JOURNAL_RECORD_ID; import static org.folio.dao.util.JournalRecordsColumns.INVOICE_LINE_NUMBER; import static org.folio.dao.util.JournalRecordsColumns.ITEM_ACTION_STATUS; import static org.folio.dao.util.JournalRecordsColumns.ITEM_ENTITY_ERROR; import static org.folio.dao.util.JournalRecordsColumns.ITEM_ENTITY_HRID; import static org.folio.dao.util.JournalRecordsColumns.ITEM_ENTITY_ID; import static org.folio.dao.util.JournalRecordsColumns.JOB_EXECUTION_ID; +import static org.folio.dao.util.JournalRecordsColumns.ORDER_ID; +import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ACTION_STATUS; +import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_ERROR; +import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_HRID; +import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_ID; import static org.folio.dao.util.JournalRecordsColumns.SOURCE_ENTITY_ERROR; import static org.folio.dao.util.JournalRecordsColumns.SOURCE_ID; import static org.folio.dao.util.JournalRecordsColumns.SOURCE_RECORD_ACTION_STATUS; @@ -122,13 +125,7 @@ import static org.folio.dao.util.JournalRecordsColumns.TOTAL_UPDATED_ITEMS; import static org.folio.dao.util.JournalRecordsColumns.TOTAL_UPDATED_ORDERS; import static org.folio.dao.util.JournalRecordsColumns.TOTAL_UPDATED_SOURCE_RECORDS; -import static org.folio.dao.util.JournalRecordsColumns.ORDER_ID; -import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ACTION_STATUS; -import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_ID; -import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_HRID; -import static org.folio.dao.util.JournalRecordsColumns.PO_LINE_ENTITY_ERROR; import static org.folio.rest.jaxrs.model.ActionStatus.UPDATED; -import static org.folio.rest.jaxrs.model.JobLogEntryDto.SourceRecordType.MARC_HOLDINGS; import static org.folio.rest.persist.PostgresClient.convertToPsqlStandard; @Repository @@ -137,6 +134,9 @@ public class JournalRecordDaoImpl implements JournalRecordDao { private static final Logger LOGGER = LogManager.getLogger(); public static final String SOURCE_RECORD_ENTITY_TYPE = "source_record_entity_type"; public static final String ORDER_ENTITY_ID = "order_entity_id"; + public static final String SOURCE_ENTITY_ERROR = "source_entity_error"; + public static final String INCOMING_RECORD_ID = "incoming_record_id"; + public static final String HOLDINGS_ENTITY_HRID = "holdings_entity_hrid"; private final Set sortableFields = Set.of("source_record_order", "action_type", "error"); private final Set jobLogEntrySortableFields = Set.of("source_record_order", "title", "source_record_action_status", "instance_action_status", "holdings_action_status", "item_action_status", "order_action_status", "invoice_action_status", "error"); @@ -239,7 +239,7 @@ public Future deleteByJobExecutionId(String jobExecutionId, String tena } @Override - public Future getJobLogEntryDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId) { + public Future getRecordProcessingLogDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId) { LOGGER.trace("getJobLogEntryDtoCollection:: Trying to get JobLogEntryDtoCollection entity by jobExecutionId = {}", jobExecutionId); if (!jobLogEntrySortableFields.contains(sortBy)) { return Future.failedFuture(new BadRequestException(format("The specified field for sorting job log entries is invalid: '%s'", sortBy))); @@ -248,7 +248,7 @@ public Future getJobLogEntryDtoCollection(String jobEx String query = format(GET_JOB_LOG_ENTRIES_BY_JOB_EXECUTION_ID_QUERY, jobExecutionId, sortBy, order, limit, offset, errorsOnly, entityType); LOGGER.trace("JournalRecordDaoImpl::getJobLogEntryDtoCollection query = {};", query); pgClientFactory.createInstance(tenantId).select(query, promise); - return promise.future().map(this::mapRowSetToJobLogDtoCollection); + return promise.future().map(this::mapRowSetToRecordProcessingLogDtoCollection); } @Override @@ -316,48 +316,73 @@ private String prepareSortingClause(String sortBy, String order) { return format(ORDER_BY_PATTERN, sortBy, order); } - private JobLogEntryDtoCollection mapRowSetToJobLogDtoCollection(RowSet rowSet) { - var jobLogEntryDtoCollection = new JobLogEntryDtoCollection() + private RecordProcessingLogDtoCollection mapRowSetToRecordProcessingLogDtoCollection(RowSet rowSet) { + var recordProcessingLogDto = new RecordProcessingLogDtoCollection() .withTotalRecords(0); rowSet.forEach(row -> - jobLogEntryDtoCollection + recordProcessingLogDto .withTotalRecords(row.getInteger(TOTAL_COUNT)) .getEntries().add(mapJobLogEntryRow(row)) ); - return jobLogEntryDtoCollection; + return recordProcessingLogDto; } - private JobLogEntryDto mapJobLogEntryRow(Row row) { - final var entityType = mapToEntityType(row.getString(SOURCE_RECORD_ENTITY_TYPE)); - final var entityHrid = row.getArrayOfStrings(HOLDINGS_ENTITY_HRID); + private RecordProcessingLogDto mapJobLogEntryRow(Row row) { + RecordProcessingLogDto recordProcessingLogSummary = new RecordProcessingLogDto(); + + List processedHoldingsInfo = new LinkedList<>(); + List processedItemInfo = new LinkedList<>(); + + final var entityType = mapToRecordProcessingEntityType(row.getString(SOURCE_RECORD_ENTITY_TYPE)); + final var holdingsEntityHrid = row.getString(HOLDINGS_ENTITY_HRID); final var holdingsActionStatus = mapNameToEntityActionStatus(row.getString(HOLDINGS_ACTION_STATUS)); - return new JobLogEntryDto() + recordProcessingLogSummary + .withSourceRecordType(entityType) .withJobExecutionId(row.getValue(JOB_EXECUTION_ID).toString()) + .withIncomingRecordId(row.getValue(INCOMING_RECORD_ID).toString()) .withSourceRecordId(row.getValue(SOURCE_ID).toString()) .withSourceRecordOrder(isEmpty(row.getString(INVOICE_ACTION_STATUS)) ? row.getInteger(SOURCE_RECORD_ORDER).toString() : row.getString(INVOICE_LINE_NUMBER)) - .withSourceRecordTitle(getJobLogEntryTitle(row.getString(TITLE), entityType, entityHrid, holdingsActionStatus)) - .withSourceRecordType(entityType) - .withHoldingsRecordHridList(ArrayUtils.isEmpty(entityHrid) ? Collections.emptyList() : Arrays.asList(entityHrid)) + .withSourceRecordTitle(getJobLogEntryTitle(row.getString(TITLE), entityType, holdingsEntityHrid, holdingsActionStatus)) .withSourceRecordActionStatus(mapNameToEntityActionStatus(row.getString(SOURCE_RECORD_ACTION_STATUS))) - .withInstanceActionStatus(mapNameToEntityActionStatus(row.getString(INSTANCE_ACTION_STATUS))) - .withHoldingsActionStatus(holdingsActionStatus) - .withItemActionStatus(mapNameToEntityActionStatus(row.getString(ITEM_ACTION_STATUS))) - .withAuthorityActionStatus(mapNameToEntityActionStatus(row.getString(AUTHORITY_ACTION_STATUS))) - .withPoLineActionStatus(mapNameToEntityActionStatus(row.getString(PO_LINE_ACTION_STATUS))) - .withInvoiceActionStatus(mapNameToEntityActionStatus(row.getString(INVOICE_ACTION_STATUS))) - .withInvoiceLineJournalRecordId(Objects.isNull(row.getValue(INVOICE_LINE_JOURNAL_RECORD_ID)) - ? null : row.getValue(INVOICE_LINE_JOURNAL_RECORD_ID).toString()) - .withError(row.getString(ERROR)); + .withError(row.getString(SOURCE_ENTITY_ERROR)) + .withRelatedInstanceInfo(constructInstanceProcessingInfo(row)) + .withRelatedAuthorityInfo(constructProcessedEntityWithSingleIdInfoBasedOnEntityType(row, + AUTHORITY_ACTION_STATUS, AUTHORITY_ENTITY_ID, null, AUTHORITY_ENTITY_ERROR)) + .withRelatedPoLineInfo(new RelatedPoLineInfo() + .withActionStatus(mapNameToEntityActionStatus(row.getString(PO_LINE_ACTION_STATUS))) + .withIdList(constructSingletonListFromColumn(row, PO_LINE_ENTITY_ID)) + .withHridList(constructSingletonListFromColumn(row, PO_LINE_ENTITY_HRID)) + .withError(row.getString(PO_LINE_ENTITY_ERROR)) + .withOrderId(row.getString(ORDER_ENTITY_ID))) + .withRelatedInvoiceInfo(constructProcessedEntityInfoBasedOnEntityType(row, + INVOICE_ACTION_STATUS, INVOICE_ENTITY_ID, INVOICE_ENTITY_HRID, INVOICE_ENTITY_ERROR)) + .withRelatedInvoiceLineInfo(constructInvoiceLineInfo(row)) + .withSourceRecordTenantId(row.getString(SOURCE_RECORD_TENANT_ID)); + + ProcessedHoldingsInfo processedHoldings = constructProcessedHoldingsInfoBasedOnEntityType(row, HOLDINGS_ACTION_STATUS, HOLDINGS_ENTITY_ID, JournalRecordsColumns.HOLDINGS_ENTITY_HRID, HOLDINGS_PERMANENT_LOCATION_ID, HOLDINGS_ENTITY_ERROR); + ProcessedItemInfo processedItem = constructProcessedItemInfoBasedOnEntityType(row, ITEM_ACTION_STATUS, ITEM_ENTITY_ID, ITEM_ENTITY_HRID, HOLDINGS_ENTITY_ID, ITEM_ENTITY_ERROR); + if (Objects.nonNull(processedHoldings.getActionStatus()) || processedItem.getActionStatus() == UPDATED) { + processedHoldingsInfo.add(processedHoldings); + } + if (Objects.nonNull(processedItem.getActionStatus())) { + processedItemInfo.add(processedItem); + } + + recordProcessingLogSummary. + withRelatedItemInfo(processedItemInfo.stream().distinct().toList()) + .withRelatedHoldingsInfo(processedHoldingsInfo.stream().distinct().toList()); + + return recordProcessingLogSummary; } - private String getJobLogEntryTitle(String title, JobLogEntryDto.SourceRecordType entityType, String[] entityHrid, + private String getJobLogEntryTitle(String title, RecordProcessingLogDto.SourceRecordType entityType, String entityHrid, org.folio.rest.jaxrs.model.ActionStatus holdingsActionStatus) { - return MARC_HOLDINGS.equals(entityType) + return RecordProcessingLogDto.SourceRecordType.MARC_HOLDINGS.equals(entityType) && isActionStatusUpdatedOrCreated(holdingsActionStatus) - ? "Holdings " + entityHrid[0] + ? "Holdings " + entityHrid : title; } @@ -378,10 +403,10 @@ private RecordProcessingLogDto mapRowSetToRecordProcessingLogDto(RowSet res recordProcessingLogSummary .withJobExecutionId(row.getValue(JOB_EXECUTION_ID).toString()) .withSourceRecordId(row.getValue(SOURCE_ID).toString()) - .withSourceRecordOrder(row.getInteger(SOURCE_RECORD_ORDER)) + .withSourceRecordOrder(row.getInteger(SOURCE_RECORD_ORDER).toString()) .withSourceRecordTitle(row.getString(TITLE)) .withSourceRecordActionStatus(mapNameToEntityActionStatus(row.getString(SOURCE_RECORD_ACTION_STATUS))) - .withError(row.getString(SOURCE_ENTITY_ERROR)) + .withError(row.getString(JournalRecordsColumns.SOURCE_ENTITY_ERROR)) .withSourceRecordTenantId(row.getString(SOURCE_RECORD_TENANT_ID)) .withRelatedInstanceInfo(constructInstanceProcessingInfo(row)) .withRelatedAuthorityInfo(constructProcessedEntityWithSingleIdInfoBasedOnEntityType(row, @@ -398,7 +423,7 @@ private RecordProcessingLogDto mapRowSetToRecordProcessingLogDto(RowSet res } resultSet.forEach(r -> { - ProcessedHoldingsInfo processedHoldings = constructProcessedHoldingsInfoBasedOnEntityType(r, HOLDINGS_ACTION_STATUS, HOLDINGS_ENTITY_ID, HOLDINGS_ENTITY_HRID, HOLDINGS_PERMANENT_LOCATION_ID, HOLDINGS_ENTITY_ERROR); + ProcessedHoldingsInfo processedHoldings = constructProcessedHoldingsInfoBasedOnEntityType(r, HOLDINGS_ACTION_STATUS, HOLDINGS_ENTITY_ID, JournalRecordsColumns.HOLDINGS_ENTITY_HRID, HOLDINGS_PERMANENT_LOCATION_ID, HOLDINGS_ENTITY_ERROR); ProcessedItemInfo processedItem = constructProcessedItemInfoBasedOnEntityType(r, ITEM_ACTION_STATUS, ITEM_ENTITY_ID, ITEM_ENTITY_HRID, HOLDINGS_ENTITY_ID, ITEM_ENTITY_ERROR); if (Objects.nonNull(processedHoldings.getActionStatus()) || processedItem.getActionStatus() == UPDATED) { processedHoldingsInfo.add(processedHoldings); @@ -474,8 +499,8 @@ private org.folio.rest.jaxrs.model.ActionStatus mapNameToEntityActionStatus(Stri return name == null ? null : org.folio.rest.jaxrs.model.ActionStatus.fromValue(name); } - private JobLogEntryDto.SourceRecordType mapToEntityType(String entityType) { - return entityType == null ? null : JobLogEntryDto.SourceRecordType.fromValue(entityType); + private RecordProcessingLogDto.SourceRecordType mapToRecordProcessingEntityType(String entityType) { + return entityType == null ? null : RecordProcessingLogDto.SourceRecordType.fromValue(entityType); } private JobExecutionSummaryDto mapRowToJobExecutionSummaryDto(Row row) { diff --git a/mod-source-record-manager-server/src/main/java/org/folio/rest/impl/MetadataProviderImpl.java b/mod-source-record-manager-server/src/main/java/org/folio/rest/impl/MetadataProviderImpl.java index 8a91fd01e..edcc865a1 100644 --- a/mod-source-record-manager-server/src/main/java/org/folio/rest/impl/MetadataProviderImpl.java +++ b/mod-source-record-manager-server/src/main/java/org/folio/rest/impl/MetadataProviderImpl.java @@ -114,7 +114,7 @@ public void getMetadataProviderJobLogEntriesByJobExecutionId(String jobExecution try { LOGGER.debug("getMetadataProviderJobLogEntriesByJobExecutionId:: jobExecutionId {}, sortBy {}, errorsOnly {}, entityType {}", jobExecutionId, sortBy, errorsOnly, entityType.name()); - journalRecordService.getJobLogEntryDtoCollection(jobExecutionId, sortBy, order.name(), errorsOnly, entityType.name(), limit, offset, tenantId) + journalRecordService.getRecordProcessingLogDtoCollection(jobExecutionId, sortBy, order.name(), errorsOnly, entityType.name(), limit, offset, tenantId) .map(GetMetadataProviderJobLogEntriesByJobExecutionIdResponse::respond200WithApplicationJson) .map(Response.class::cast) .otherwise(ExceptionHelper::mapExceptionToResponse) diff --git a/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordService.java b/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordService.java index b7e937e23..58d324e5d 100644 --- a/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordService.java +++ b/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordService.java @@ -3,9 +3,9 @@ import io.vertx.core.Future; import org.folio.rest.jaxrs.model.JobExecutionSummaryDto; import org.folio.rest.jaxrs.model.JournalRecord; -import org.folio.rest.jaxrs.model.JobLogEntryDtoCollection; import org.folio.rest.jaxrs.model.JournalRecordCollection; import org.folio.rest.jaxrs.model.RecordProcessingLogDto; +import org.folio.rest.jaxrs.model.RecordProcessingLogDtoCollection; import java.util.List; import java.util.Optional; @@ -42,13 +42,13 @@ public interface JournalRecordService { * @param sortBy sorting criteria * @param order sorting direction * @param errorsOnly filtering by error field - * @param errorsOnly filtering by entity type + * @param entityType filtering by entity type * @param limit limit * @param offset offset * @param tenantId tenantId * @return future with JobLogEntryDto collection */ - Future getJobLogEntryDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId); + Future getRecordProcessingLogDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId); /** * Searches for RecordProcessingLogDto entity by jobExecutionId and recordId diff --git a/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordServiceImpl.java b/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordServiceImpl.java index 99b3464b7..d0bf68848 100644 --- a/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordServiceImpl.java +++ b/mod-source-record-manager-server/src/main/java/org/folio/services/JournalRecordServiceImpl.java @@ -4,9 +4,9 @@ import org.folio.dao.JournalRecordDao; import org.folio.rest.jaxrs.model.JobExecutionSummaryDto; import org.folio.rest.jaxrs.model.JournalRecord; -import org.folio.rest.jaxrs.model.JobLogEntryDtoCollection; import org.folio.rest.jaxrs.model.JournalRecordCollection; import org.folio.rest.jaxrs.model.RecordProcessingLogDto; +import org.folio.rest.jaxrs.model.RecordProcessingLogDtoCollection; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -38,8 +38,8 @@ public Future getJobExecutionJournalRecords(String jobE } @Override - public Future getJobLogEntryDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId) { - return journalRecordDao.getJobLogEntryDtoCollection(jobExecutionId, sortBy, order, errorsOnly, entityType, limit, offset, tenantId); + public Future getRecordProcessingLogDtoCollection(String jobExecutionId, String sortBy, String order, boolean errorsOnly, String entityType, int limit, int offset, String tenantId) { + return journalRecordDao.getRecordProcessingLogDtoCollection(jobExecutionId, sortBy, order, errorsOnly, entityType, limit, offset, tenantId); } @Override diff --git a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql index 7e4f04bb9..483d4d8da 100644 --- a/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql +++ b/mod-source-record-manager-server/src/main/resources/templates/db_scripts/create_get_job_log_entries_function.sql @@ -24,11 +24,15 @@ DROP FUNCTION IF EXISTS get_job_log_entries(uuid,text,text,bigint,bigint); DROP FUNCTION IF EXISTS get_job_log_entries(uuid,text,text,bigint,bigint,boolean,text); -- Script to create function to get data import job log entries (jobLogEntry). -CREATE OR REPLACE FUNCTION get_job_log_entries(jobExecutionId uuid, sortingField text, sortingDir text, limitVal bigint, offsetVal bigint, errorsOnly boolean, entityType text) - RETURNS TABLE(job_execution_id uuid, source_id uuid, source_record_order integer, invoiceline_number text, title text, - source_record_action_status text, instance_action_status text, holdings_action_status text, item_action_status text, - authority_action_status text, po_line_action_status text, invoice_action_status text, error text, total_count bigint, - invoice_line_journal_record_id uuid, source_record_entity_type text, holdings_entity_hrid text[], source_record_order_array integer[]) +CREATE OR REPLACE FUNCTION diku_mod_source_record_manager.get_job_log_entries(jobExecutionId uuid, sortingField text, sortingDir text, limitVal bigint, offsetVal bigint, errorsOnly boolean, entityType text) + RETURNS TABLE(job_execution_id uuid, incoming_record_id uuid, source_id uuid, source_record_order integer, invoiceline_number text, title text, + source_record_action_status text, source_entity_error text, source_record_tenant_id text,instance_action_status text, instance_entity_id text, instance_entity_hrid text, instance_entity_error text, + instance_entity_tenant_id text, holdings_action_status text, holdings_entity_hrid text, holdings_entity_id text, holdings_permanent_location_id text, + holdings_entity_error text, item_action_status text, item_entity_hrid text, item_entity_id text, item_entity_error text, authority_action_status text, + authority_entity_id text, authority_entity_error text, po_line_action_status text, po_line_entity_id text, po_line_entity_hrid text, po_line_entity_error text, + order_entity_id text, invoice_action_status text, invoice_entity_id text[], invoice_entity_hrid text[], invoice_entity_error text, invoice_line_action_status text, + invoice_line_entity_id text, invoice_line_entity_hrid text, invoice_line_entity_error text, total_count bigint, + invoice_line_journal_record_id uuid, source_record_entity_type text, source_record_order_array integer[]) AS $$ DECLARE @@ -45,98 +49,576 @@ BEGIN END IF; RETURN QUERY EXECUTE format(' -SELECT records_actions.job_execution_id, records_actions.source_id, records_actions.source_record_order, '''' as invoiceline_number, +WITH records_actions AS + (SELECT journal_records.source_id, + journal_records.source_record_order, + journal_records.job_execution_id, + array_agg(action_type + ORDER BY array_position(array[''MATCH'', ''NON_MATCH'', ''MODIFY'', ''UPDATE'', ''CREATE''], action_type)) FILTER ( + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')) AS marc_actions, + count(journal_records.source_id) FILTER ( + WHERE (entity_type = ''MARC_BIBLIOGRAPHIC'' + OR entity_type = ''MARC_HOLDINGS'' + OR entity_type = ''MARC_AUTHORITY'') + AND journal_records.error != '''') AS marc_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH'', ''MATCH''], action_type)) FILTER ( + WHERE entity_type = ''INSTANCE'' + AND (entity_id IS NOT NULL + OR action_type = ''NON_MATCH'')) AS instance_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''INSTANCE'' + AND journal_records.error != '''') AS instance_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''HOLDINGS'') AS holdings_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''HOLDINGS'' + AND journal_records.error != '''') AS holdings_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''ITEM'') AS item_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''ITEM'' + AND journal_records.error != '''') AS item_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''AUTHORITY'') AS authority_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''AUTHORITY'' + AND journal_records.error != '''') AS authority_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''PO_LINE'') AS po_line_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''PO_LINE'' + AND journal_records.error != '''') AS po_line_errors_number, + count(journal_records.source_id) OVER () AS total_count, + (array_agg(journal_records.entity_type) FILTER ( + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')))[1] AS source_record_entity_type, + array_agg(journal_records.entity_hrid) FILTER ( + WHERE entity_hrid !='''' + AND entity_type = ''HOLDINGS'') AS holdings_entity_hrid + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'' + AND entity_type in (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'', + ''INSTANCE'', + ''HOLDINGS'', + ''ITEM'', + ''AUTHORITY'', + ''PO_LINE'') + GROUP BY journal_records.source_id, + journal_records.source_record_order, + journal_records.job_execution_id + HAVING count(journal_records.source_id) FILTER ( + WHERE (%3$L = ''ALL'' + OR entity_type = ANY(%4$L)) + AND (NOT %2$L + OR journal_records.error <> '''')) > 0), + temp_result AS + (SELECT id, + job_execution_id, + source_id, + entity_type, + entity_id, + entity_hrid, + CASE + WHEN error_max != '''' + OR action_type = ''NON_MATCH'' THEN ''DISCARDED'' + WHEN action_type = ''CREATE'' THEN ''CREATED'' + WHEN action_type IN (''UPDATE'', + ''MODIFY'') THEN ''UPDATED'' + END AS action_type, + action_status, + action_date, + source_record_order, + error, + title, + tenant_id, + instance_id, + holdings_id, + order_id, + permanent_location_id + FROM journal_records + INNER JOIN + (SELECT entity_type AS entity_type_max, + entity_id AS entity_id_max, + action_status AS action_status_max, + max(error) AS error_max, + (array_agg(id + ORDER BY array_position(array[''CREATE'', ''UPDATE'', ''MODIFY'', ''NON_MATCH''], action_type)))[1] AS id_max + FROM journal_records + WHERE job_execution_id = ''%1$s'' + AND entity_type NOT IN (''EDIFACT'', + ''INVOICE'') + GROUP BY entity_type, + entity_id, + action_status) AS action_type_by_source ON journal_records.id = action_type_by_source.id_max), + instances AS + (SELECT action_type, + entity_id, + source_id, + entity_hrid, + error, + job_execution_id, + title, + source_record_order, + tenant_id + FROM temp_result + WHERE entity_type = ''INSTANCE'' ), + holdings AS + (SELECT action_type, + entity_id, + entity_hrid, + error, + instance_id, + permanent_location_id, + temp_result.job_execution_id, + temp_result.source_id, + temp_result.title, + temp_result.source_record_order + FROM temp_result + WHERE entity_type = ''HOLDINGS'' ), + items AS + (SELECT action_type, + entity_id, + holdings_id, + entity_hrid, + error, + instance_id, + temp_result.job_execution_id, + temp_result.source_id, + temp_result.title, + temp_result.source_record_order + FROM temp_result + WHERE entity_type = ''ITEM'' ), + po_lines AS + (SELECT action_type, + entity_id, + entity_hrid, + temp_result.source_id, + error, + order_id, + temp_result.job_execution_id, + temp_result.title, + temp_result.source_record_order + FROM temp_result + WHERE entity_type = ''PO_LINE'' ), + authorities AS + (SELECT action_type, + entity_id, + temp_result.source_id, + error, + temp_result.job_execution_id, + temp_result.title, + temp_result.source_record_order + FROM temp_result + WHERE entity_type = ''AUTHORITY'' ) +SELECT records_actions.job_execution_id AS job_execution_id, + records_actions.source_id AS source_id, + records_actions.source_id AS incoming_record_id, + records_actions.source_record_order AS source_record_order, + '''' AS invoiceline_number, rec_titles.title, CASE - WHEN marc_errors_number != 0 OR marc_actions[array_length(marc_actions, 1)] = ''NON_MATCH'' THEN ''DISCARDED'' - WHEN marc_actions[array_length(marc_actions, 1)] = ''CREATE'' THEN ''CREATED'' - WHEN marc_actions[array_length(marc_actions, 1)] IN (''UPDATE'', ''MODIFY'') THEN ''UPDATED'' - END AS source_record_action_status, + WHEN marc_errors_number != 0 + OR marc_actions[array_length(marc_actions, 1)] = ''NON_MATCH'' THEN ''DISCARDED'' + WHEN marc_actions[array_length(marc_actions, 1)] = ''CREATE'' THEN ''CREATED'' + WHEN marc_actions[array_length(marc_actions, 1)] IN (''UPDATE'', + ''MODIFY'') THEN ''UPDATED'' + END AS source_record_action_status, + NULL AS source_entity_error, + NULL AS source_record_tenant_id, get_entity_status(instance_actions, instance_errors_number) AS instance_action_status, + instance_info.instance_entity_id AS instance_entity_id, + instance_info.instance_entity_hrid AS instance_entity_hrid, + instance_info.instance_entity_error AS instance_entity_error, + instance_info.instance_entity_tenant_id AS instance_entity_tenant_id, get_entity_status(holdings_actions, holdings_errors_number) AS holdings_action_status, + holdings_info.holdings_entity_id AS holdings_entity_id, + holdings_info.holdings_entity_hrid AS holdings_entity_hrid, + holdings_info.holdings_permanent_location_id AS holdings_permanent_location_id, + holdings_info.holdings_entity_error AS holdings_entity_error, get_entity_status(item_actions, item_errors_number) AS item_action_status, + items_info.items_entity_id AS item_entity_id, + items_info.items_entity_hrid AS item_entity_hrid, + items_info.items_entity_error AS item_entity_error, get_entity_status(authority_actions, authority_errors_number) AS authority_action_status, + authority_info.authority_entity_id AS authority_entity_id, + authority_info.authority_entity_error AS authority_entity_error, get_entity_status(po_line_actions, po_line_errors_number) AS po_line_action_status, - null AS invoice_action_status, rec_errors.error, records_actions.total_count, - null AS invoiceLineJournalRecordId, - records_actions.source_record_entity_type, - records_actions.holdings_entity_hrid, - ARRAY[records_actions.source_record_order] AS source_record_order_array -FROM ( - SELECT journal_records.source_id, journal_records.source_record_order, journal_records.job_execution_id, - array_agg(action_type ORDER BY array_position(array[''MATCH'', ''NON_MATCH'', ''MODIFY'', ''UPDATE'', ''CREATE''], action_type)) FILTER (WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')) AS marc_actions, - count(journal_records.source_id) FILTER (WHERE (entity_type = ''MARC_BIBLIOGRAPHIC'' OR entity_type = ''MARC_HOLDINGS'' OR entity_type = ''MARC_AUTHORITY'') AND journal_records.error != '''') AS marc_errors_number, - array_agg(action_type ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH'', ''MATCH''], action_type)) FILTER (WHERE entity_type = ''INSTANCE'' AND (entity_id IS NOT NULL OR action_type = ''NON_MATCH'')) AS instance_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''INSTANCE'' AND journal_records.error != '''') AS instance_errors_number, - array_agg(action_type ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER (WHERE entity_type = ''HOLDINGS'') AS holdings_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''HOLDINGS'' AND journal_records.error != '''') AS holdings_errors_number, - array_agg(action_type ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER (WHERE entity_type = ''ITEM'') AS item_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''ITEM'' AND journal_records.error != '''') AS item_errors_number, - array_agg(action_type ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER (WHERE entity_type = ''AUTHORITY'') AS authority_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''AUTHORITY'' AND journal_records.error != '''') AS authority_errors_number, - array_agg(action_type ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER (WHERE entity_type = ''PO_LINE'') AS po_line_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''PO_LINE'' AND journal_records.error != '''') AS po_line_errors_number, - count(journal_records.source_id) OVER () AS total_count, - (array_agg(journal_records.entity_type) FILTER (WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')))[1] AS source_record_entity_type, - array_agg(journal_records.entity_hrid) FILTER (WHERE entity_hrid !='''' and entity_type = ''HOLDINGS'') as holdings_entity_hrid - FROM journal_records - WHERE journal_records.job_execution_id = ''%1$s'' and - entity_type in (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'', ''INSTANCE'', ''HOLDINGS'', ''ITEM'', ''AUTHORITY'', ''PO_LINE'') - GROUP BY journal_records.source_id, journal_records.source_record_order, journal_records.job_execution_id - HAVING count(journal_records.source_id) FILTER (WHERE (%3$L = ''ALL'' or entity_type = ANY(%4$L)) AND (NOT %2$L or journal_records.error <> '''')) > 0 - ) AS records_actions - LEFT JOIN (SELECT journal_records.source_id, - CASE - WHEN COUNT(*) = 1 THEN array_to_string(array_agg(journal_records.error), '', '') - ELSE ''['' || array_to_string(array_agg(journal_records.error), '', '') || '']'' - END AS error - FROM journal_records - WHERE journal_records.job_execution_id = ''%1$s'' AND journal_records.error != '''' GROUP BY journal_records.source_id) AS rec_errors - ON rec_errors.source_id = records_actions.source_id - LEFT JOIN (SELECT journal_records.source_id, journal_records.title - FROM journal_records - WHERE journal_records.job_execution_id = ''%1$s'') AS rec_titles - ON rec_titles.source_id = records_actions.source_id AND rec_titles.title IS NOT NULL - + po_lines_info.po_lines_entity_id AS po_lines_entity_id, + po_lines_info.po_lines_entity_hrid AS po_lines_entity_hrid, + po_lines_info.po_lines_entity_error AS po_lines_entity_error, + NULL AS order_entity_id, + NULL AS invoice_action_status, + NULL::text[] AS invoice_entity_id, + NULL::text[] AS invoice_entity_hrid, + NULL AS invoice_entity_error, + NULL AS invoice_line_action_status, + NULL AS invoice_line_entity_id, + NULL AS invoice_line_entity_hrid, + NULL AS invoice_line_entity_error, + records_actions.total_count, + NULL::UUID AS invoice_line_journal_record_id, + records_actions.source_record_entity_type, ARRAY[records_actions.source_record_order] AS source_record_order_array +FROM + (SELECT journal_records.source_id, + journal_records.source_record_order, + journal_records.job_execution_id, + array_agg(action_type + ORDER BY array_position(array[''MATCH'', ''NON_MATCH'', ''MODIFY'', ''UPDATE'', ''CREATE''], action_type)) FILTER ( + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')) AS marc_actions, + count(journal_records.source_id) FILTER ( + WHERE (entity_type = ''MARC_BIBLIOGRAPHIC'' + OR entity_type = ''MARC_HOLDINGS'' + OR entity_type = ''MARC_AUTHORITY'') + AND journal_records.error != '''') AS marc_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH'', ''MATCH''], action_type)) FILTER ( + WHERE entity_type = ''INSTANCE'' + AND (entity_id IS NOT NULL + OR action_type = ''NON_MATCH'')) AS instance_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''INSTANCE'' + AND journal_records.error != '''') AS instance_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''HOLDINGS'') AS holdings_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''HOLDINGS'' + AND journal_records.error != '''') AS holdings_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''ITEM'') AS item_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''ITEM'' + AND journal_records.error != '''') AS item_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''AUTHORITY'') AS authority_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''AUTHORITY'' + AND journal_records.error != '''') AS authority_errors_number, + array_agg(action_type + ORDER BY array_position(array[''CREATE'', ''MODIFY'', ''UPDATE'', ''NON_MATCH''], action_type)) FILTER ( + WHERE entity_type = ''PO_LINE'') AS po_line_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''PO_LINE'' + AND journal_records.error != '''') AS po_line_errors_number, + count(journal_records.source_id) OVER () AS total_count, + (array_agg(journal_records.entity_type) FILTER ( + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', ''MARC_HOLDINGS'', ''MARC_AUTHORITY'')))[1] AS source_record_entity_type, + array_agg(journal_records.entity_hrid) FILTER ( + WHERE entity_hrid !='''' + AND entity_type = ''HOLDINGS'') AS holdings_entity_hrid + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'' + AND entity_type in (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'', + ''INSTANCE'', + ''HOLDINGS'', + ''ITEM'', + ''AUTHORITY'', + ''PO_LINE'') + GROUP BY journal_records.source_id, + journal_records.source_record_order, + journal_records.job_execution_id + HAVING count(journal_records.source_id) FILTER ( + WHERE (%3$L = ''ALL'' + OR entity_type = ANY(%4$L)) + AND (NOT %2$L + OR journal_records.error <> '''')) > 0) AS records_actions + LEFT JOIN + (SELECT instances.source_id AS source_id, + instances.entity_id AS instance_entity_id, + instances.entity_hrid AS instance_entity_hrid, + instances.error AS instance_entity_error, + instances.tenant_id AS instance_entity_tenant_id + FROM + (SELECT temp_result.job_execution_id, + entity_id, + title, + source_record_order, + action_type, + error, + source_id, + tenant_id + FROM temp_result + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'') ) AS marc + LEFT JOIN instances ON marc.source_id = instances.source_id) AS instance_info ON instance_info.source_id = records_actions.source_id + LEFT JOIN + (SELECT holdings.source_id AS source_id, + holdings.entity_id AS holdings_entity_id, + holdings.entity_hrid AS holdings_entity_hrid, + holdings.permanent_location_id AS holdings_permanent_location_id, + holdings.error AS holdings_entity_error + FROM + (SELECT temp_result.job_execution_id, + entity_id, + title, + source_record_order, + action_type, + error, + source_id, + tenant_id + FROM temp_result + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'') ) AS marc + LEFT JOIN holdings ON marc.source_id = holdings.source_id) AS holdings_info ON holdings_info.source_id = records_actions.source_id + LEFT JOIN + (SELECT items.source_id AS source_id, + items.entity_id AS items_entity_id, + items.entity_hrid AS items_entity_hrid, + items.error AS items_entity_error + FROM + (SELECT temp_result.job_execution_id, + entity_id, + title, + source_record_order, + action_type, + error, + source_id, + tenant_id + FROM temp_result + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'') ) AS marc + LEFT JOIN items ON marc.source_id = items.source_id) AS items_info ON items_info.source_id = records_actions.source_id + LEFT JOIN + (SELECT po_lines.source_id AS source_id, + po_lines.entity_id AS po_lines_entity_id, + po_lines.entity_hrid AS po_lines_entity_hrid, + po_lines.error AS po_lines_entity_error + FROM + (SELECT temp_result.job_execution_id, + entity_id, + title, + source_record_order, + action_type, + error, + source_id, + tenant_id + FROM temp_result + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'') ) AS marc + LEFT JOIN po_lines ON marc.source_id = po_lines.source_id) AS po_lines_info ON po_lines_info.source_id = records_actions.source_id + LEFT JOIN + (SELECT authorities.source_id AS source_id, + authorities.entity_id AS authority_entity_id, + authorities.error AS authority_entity_error + FROM + (SELECT temp_result.job_execution_id, + entity_id, + title, + source_record_order, + action_type, + error, + source_id, + tenant_id + FROM temp_result + WHERE entity_type IN (''MARC_BIBLIOGRAPHIC'', + ''MARC_HOLDINGS'', + ''MARC_AUTHORITY'') ) AS marc + LEFT JOIN authorities ON marc.source_id = authorities.source_id) AS authority_info ON authority_info.source_id = records_actions.source_id + LEFT JOIN + (SELECT journal_records.source_id, + CASE + WHEN COUNT(*) = 1 THEN array_to_string(array_agg(journal_records.error), '', '') + ELSE ''['' || array_to_string(array_agg(journal_records.error), '', '') || '']'' + END AS error + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'' + AND journal_records.error != '''' + GROUP BY journal_records.source_id) AS rec_errors ON rec_errors.source_id = records_actions.source_id + LEFT JOIN + (SELECT journal_records.source_id, + journal_records.title + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'') AS rec_titles ON rec_titles.source_id = records_actions.source_id + AND rec_titles.title IS NOT NULL UNION - -SELECT records_actions.job_execution_id, records_actions.source_id, source_record_order, entity_hrid as invoiceline_number, title, +SELECT records_actions.job_execution_id AS job_execution_id, + records_actions.source_id AS source_id, + records_actions.source_id AS incoming_record_id, + records_actions.source_record_order AS source_record_order, + entity_hrid AS invoiceline_number, + records_actions.title, CASE - WHEN marc_errors_number != 0 OR marc_actions[array_length(marc_actions, 1)] = ''NON_MATCH'' THEN ''DISCARDED'' - WHEN marc_actions[array_length(marc_actions, 1)] = ''CREATE'' THEN ''CREATED'' - WHEN marc_actions[array_length(marc_actions, 1)] IN (''UPDATE'', ''MODIFY'') THEN ''UPDATED'' - END AS source_record_action_status, - null AS instance_action_status, - null AS holdings_action_status, - null AS item_action_status, - null AS authority_action_status, - null AS po_line_action_status, + WHEN marc_errors_number != 0 + OR marc_actions[array_length(marc_actions, 1)] = ''NON_MATCH'' THEN ''DISCARDED'' + WHEN marc_actions[array_length(marc_actions, 1)] = ''CREATE'' THEN ''CREATED'' + WHEN marc_actions[array_length(marc_actions, 1)] IN (''UPDATE'', + ''MODIFY'') THEN ''UPDATED'' + END AS source_record_action_status, + NULL AS source_entity_error, + NULL AS source_record_tenant_id, + NULL AS instance_action_status, + NULL AS instance_entity_id, + NULL AS instance_entity_hrid, + NULL AS instance_entity_error, + NULL AS instance_entity_tenant_id, + NULL AS holdings_action_status, + NULL AS holdings_entity_hrid, + NULL AS holdings_entity_id, + NULL AS holdings_permanent_location_id, + NULL AS holdings_entity_error, + NULL AS item_action_status, + NULL AS item_entity_hrid, + NULL AS item_entity_id, + NULL AS item_entity_error, + NULL AS authority_action_status, + NULL AS authority_entity_id, + NULL AS authority_entity_error, + NULL AS po_line_action_status, + NULL AS po_line_entity_id, + NULL AS po_line_entity_hrid, + NULL AS po_line_entity_error, + NULL AS order_entity_id, get_entity_status(invoice_actions, invoice_errors_number) AS invoice_action_status, - error, + invoice.invoice_entity_id, + invoice.invoice_entity_hrid, + invoice.invoice_entity_error, + invoice.invoice_line_action_status, + invoice.invoice_line_entity_id, + invoice.invoice_line_entity_hrid, + invoice.invoice_line_entity_error, records_actions.total_count, - invoiceLineJournalRecordId, + invoiceLineJournalRecordId AS invoice_line_journal_record_id, records_actions.source_record_entity_type, - records_actions.holdings_entity_hrid, CASE - WHEN get_entity_status(invoice_actions, invoice_errors_number) IS NOT null THEN string_to_array(entity_hrid, ''-'')::int[] - ELSE ARRAY[source_record_order] - END AS source_record_order_array -FROM ( - SELECT journal_records.source_id, journal_records.job_execution_id, source_record_order, entity_hrid, title, error, - array[]::varchar[] AS marc_actions, - cast(0 as integer) AS marc_errors_number, - array_agg(action_type) FILTER (WHERE entity_type = ''INVOICE'') AS invoice_actions, - count(journal_records.source_id) FILTER (WHERE entity_type = ''INVOICE'' AND journal_records.error != '''') AS invoice_errors_number, - count(journal_records.source_id) OVER () AS total_count, - id AS invoiceLineJournalRecordId, - (array_agg(entity_type) FILTER (WHERE entity_type IN (''EDIFACT'')))[1] AS source_record_entity_type, - array[]::varchar[] as holdings_entity_hrid - FROM journal_records - WHERE journal_records.job_execution_id = ''%1$s'' and entity_type = ''INVOICE'' and title != ''INVOICE'' - GROUP BY journal_records.source_id, journal_records.source_record_order, journal_records.job_execution_id, - entity_hrid, title, error, id - HAVING count(journal_records.source_id) FILTER (WHERE (%3$L IN (''ALL'', ''INVOICE'')) AND (NOT %2$L or journal_records.error <> '''')) > 0 - ) AS records_actions + WHEN get_entity_status(invoice_actions, invoice_errors_number) IS NOT NULL THEN string_to_array(entity_hrid, ''-'')::int[] + ELSE ARRAY[records_actions.source_record_order] + END AS source_record_order_array +FROM + (SELECT journal_records.source_id, + journal_records.job_execution_id, + source_record_order, + entity_hrid, + title, + error, array[]::varchar[] AS marc_actions, + cast(0 AS integer) AS marc_errors_number, + array_agg(action_type) FILTER ( + WHERE entity_type = ''INVOICE'') AS invoice_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.error != '''') AS invoice_errors_number, + count(journal_records.source_id) OVER () AS total_count, + id AS invoiceLineJournalRecordId, + (array_agg(entity_type) FILTER ( + WHERE entity_type IN (''EDIFACT'')))[1] AS source_record_entity_type, array[]::varchar[] AS holdings_entity_hrid + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'' + AND entity_type = ''INVOICE'' + AND title != ''INVOICE'' + GROUP BY journal_records.source_id, + journal_records.source_record_order, + journal_records.job_execution_id, + entity_hrid, + title, + error, + id + HAVING count(journal_records.source_id) FILTER ( + WHERE (%3$L IN (''ALL'', ''INVOICE'')) + AND (NOT %2$L + OR journal_records.error <> '''')) > 0) AS records_actions + LEFT JOIN LATERAL + (SELECT invoice_line_info.job_execution_id, + invoice_line_info.source_id, + records_actions.source_record_order, + invoice_line_info.title, + CASE + WHEN edifact_errors_number != 0 THEN ''DISCARDED'' + WHEN edifact_actions[array_length(edifact_actions, 1)] = ''CREATE'' THEN ''CREATED'' + END AS source_record_action_status, + records_actions.source_record_error[1], records_actions.source_record_tenant_id, + NULL AS instance_action_status, + NULL AS instance_entity_id, + NULL AS instance_entity_hrid, + NULL AS instance_entity_error, + NULL AS instance_entity_tenant_id, + NULL AS holdings_action_status, + NULL AS holdings_entity_hrid, + NULL AS holdings_entity_id, + NULL AS holdings_permanent_location_id, + NULL AS holdings_entity_error, + NULL AS item_action_status, + NULL AS item_entity_hrid, + NULL AS item_entity_id, + NULL AS item_entity_error, + NULL AS authority_action_status, + NULL AS authority_entity_id, + NULL AS authority_entity_error, + NULL AS po_line_action_status, + NULL AS po_line_entity_id, + NULL AS po_line_entity_hrid, + NULL AS po_line_entity_error, + NULL AS order_entity_id, + get_entity_status(records_actions.invoice_actions, records_actions.invoice_errors_number) AS invoice_action_status, + records_actions.invoice_entity_id, + records_actions.invoice_entity_hrid, + records_actions.invoice_entity_error[1], CASE + WHEN action_status = ''ERROR'' THEN ''DISCARDED'' + WHEN inv_line_actions = ''CREATE'' THEN ''CREATED'' + END AS invoice_line_action_status, + invoice_line_info.invoice_line_entity_id, + invoice_line_info.invoice_line_entity_hrid, + invoice_line_info.invoice_line_entity_error + FROM + (SELECT journal_records.source_id, + journal_records.job_execution_id, + journal_records.title, + journal_records.action_type AS inv_line_actions, + action_status, + entity_hrid AS invoice_line_entity_hrid, + entity_id AS invoice_line_entity_id, + error AS invoice_line_entity_error + FROM journal_records + WHERE journal_records.job_execution_id = ''%1$s'' + AND journal_records.entity_type = ''INVOICE'' + AND journal_records.title != ''INVOICE'' ) AS invoice_line_info + LEFT JOIN LATERAL + (SELECT journal_records.source_id, + journal_records.source_record_order, + array_agg(action_type) FILTER ( + WHERE entity_type = ''EDIFACT'') AS edifact_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''EDIFACT'' + AND journal_records.error != '''') AS edifact_errors_number, + array_agg(error) FILTER ( + WHERE entity_type = ''EDIFACT'') AS source_record_error, + journal_records.tenant_id AS source_record_tenant_id, + array_agg(action_type) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.title = ''INVOICE'') AS invoice_actions, + count(journal_records.source_id) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.title = ''INVOICE'' + AND journal_records.error != '''') AS invoice_errors_number, + array_agg(entity_hrid) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.title = ''INVOICE'') AS invoice_entity_hrid, + array_agg(entity_id) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.title = ''INVOICE'') AS invoice_entity_id, + array_agg(error) FILTER ( + WHERE entity_type = ''INVOICE'' + AND journal_records.title = ''INVOICE'') AS invoice_entity_error + FROM journal_records + WHERE journal_records.source_id = invoice_line_info.source_id + AND (entity_type = ''EDIFACT'' + OR journal_records.title = ''INVOICE'') + GROUP BY journal_records.source_id, + journal_records.job_execution_id, + journal_records.source_record_order, + journal_records.tenant_id) AS records_actions ON TRUE) AS invoice ON TRUE ORDER BY %5$I %6$s LIMIT %7$s OFFSET %8$s;', jobExecutionId, errorsOnly, entityType, v_entityAttribute, v_sortingField, sortingDir, limitVal, offsetVal); diff --git a/ramls/metadata-provider.raml b/ramls/metadata-provider.raml index dcb51ade4..fab04351d 100644 --- a/ramls/metadata-provider.raml +++ b/ramls/metadata-provider.raml @@ -17,9 +17,8 @@ types: errors: !include raml-storage/raml-util/schemas/errors.schema jobExecutionSourceChunk: !include raml-storage/schemas/mod-source-record-manager/jobExecutionSourceChunk.json journalRecordCollection: !include raml-storage/schemas/mod-source-record-manager/journalRecordCollection.json - jobLogEntryDto: !include raml-storage/schemas/dto/jobLogEntryDto.json - jobLogEntryDtoCollection: !include raml-storage/schemas/dto/jobLogEntryDtoCollection.json - recordProcessingLogDto: !include raml-storage/schemas/dto/recordProcessingLogDto.json + recordProcessingLogDto: !include tmpSchemas/recordProcessingLogDto.json + recordProcessingLogDtoCollection: !include tmpSchemas/recordProcessingLogDtoCollection.json jobExecutionSummaryDto: !include raml-storage/schemas/dto/jobExecutionSummaryDto.json jobProfileInfoCollection: !include raml-storage/schemas/common/profileInfoCollection.json jobExecutionUserInfoCollection: !include raml-storage/schemas/dto/jobExecutionUserInfoCollection.json @@ -177,7 +176,7 @@ resourceTypes: 200: body: application/json: - type: jobLogEntryDtoCollection + type: recordProcessingLogDtoCollection 400: description: "Bad request" body: diff --git a/ramls/tmpSchemas/recordProcessingLogDto.json b/ramls/tmpSchemas/recordProcessingLogDto.json new file mode 100644 index 000000000..08253a71b --- /dev/null +++ b/ramls/tmpSchemas/recordProcessingLogDto.json @@ -0,0 +1,130 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Record processing log DTO schema", + "javaType": "org.folio.rest.jaxrs.model.RecordProcessingLogDto", + "type": "object", + "additionalProperties": false, + "properties": { + "jobExecutionId": { + "description": "UUID", + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + }, + "incomingRecordId ": { + "description": "Incoming record identifier, UUID", + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + }, + "sourceRecordId": { + "description": "SRS record identifier, UUID", + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + }, + "sourceRecordOrder": { + "description": "Order of a record in imported file", + "type": "string" + }, + "sourceRecordTitle": { + "description": "Title from record", + "type": "string" + }, + "sourceRecordActionStatus": { + "description": "Status of action performed above a record", + "$ref": "../raml-storage/schemas/dto/actionStatus.json" + }, + "sourceRecordType": { + "description": "Type of entity", + "type": "string", + "$ref": "../raml-storage/schemas/dto/recordType.json" + }, + "error": { + "description": "Error message", + "type": "string" + }, + "sourceRecordTenantId": { + "description": "Id of tenant in which action was performed over record", + "type": "string" + }, + "relatedInstanceInfo": { + "description": "Information about instance associated with source record", + "$ref": "../raml-storage/schemas/dto/processedEntityInfo.json" + }, + "relatedHoldingsInfo": { + "description": "Information about holdings associated with source record", + "type": "array", + "items": { + "$ref": "../raml-storage/schemas/dto/processedHoldingsInfo.json" + } + }, + "relatedItemInfo": { + "description": "Information about item associated with source record", + "type": "array", + "items": { + "$ref": "../raml-storage/schemas/dto/processedItemInfo.json" + } + }, + "relatedAuthorityInfo": { + "description": "Information about authority associated with source record", + "$ref": "../raml-storage/schemas/dto/processedEntityInfo.json" + }, + "relatedPoLineInfo": { + "description": "Information about poLine associated with source record", + "properties": { + "actionStatus": { + "description": "Status of action performed above folio entity, value MULTIPLE if multiple entities of the same type were affected", + "$ref": "../raml-storage/schemas/dto/actionStatus.json" + }, + "idList": { + "description": "Id of entities which were affected during data import process", + "type": "array", + "items": { + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + } + }, + "hridList": { + "description": "Human readable id of entities which were affected during data import process", + "type": "array", + "items": { + "type": "string" + } + }, + "error": { + "description": "Error message", + "type": "string" + }, + "orderId": { + "description": "Order identifier, UUID", + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + } + } + }, + "relatedInvoiceInfo": { + "description": "Information about invoice associated with source record", + "$ref": "../raml-storage/schemas/dto/processedEntityInfo.json" + }, + "relatedInvoiceLineInfo": { + "description": "Information about invoice line associated with source record", + "type": "object", + "additionalProperties": false, + "properties": { + "actionStatus": { + "description": "Status of action performed with invoice line", + "$ref": "../raml-storage/schemas/dto/actionStatus.json" + }, + "id": { + "description": "Invoice line identifier, UUID", + "$ref": "../raml-storage/raml-util/schemas/uuid.schema" + }, + "fullInvoiceLineNumber": { + "description": "Consists of vendor invoice number and invoice line sequence number", + "type": "string" + }, + "error": { + "description": "Error message", + "type": "string" + } + } + } + }, + "required": [ + "jobExecutionId", + "sourceRecordOrder" + ] +} diff --git a/ramls/tmpSchemas/recordProcessingLogDtoCollection.json b/ramls/tmpSchemas/recordProcessingLogDtoCollection.json new file mode 100644 index 000000000..9384e6ac4 --- /dev/null +++ b/ramls/tmpSchemas/recordProcessingLogDtoCollection.json @@ -0,0 +1,27 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "description": "Collection of record processing logs", + "type": "object", + "additionalProperties": false, + "properties": { + "entries": { + "description": "List of of record processing logs", + "type": "array", + "id": "entries", + "items": { + "$ref": "recordProcessingLogDto.json" + } + }, + "totalRecords": { + "description": "Total number of records in collection", + "type": "integer" + } + }, + "excludedFromEqualsAndHashCode": [ + "totalRecords" + ], + "required": [ + "entries", + "totalRecords" + ] +}