Skip to content

Commit

Permalink
MODSOURCE-824
Browse files Browse the repository at this point in the history
  • Loading branch information
VRohach committed Nov 27, 2024
1 parent 58078f9 commit 4e5baa9
Show file tree
Hide file tree
Showing 7 changed files with 641 additions and 277 deletions.
5 changes: 5 additions & 0 deletions NEWS.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
## 2024-11-27 5.9.3
* [MODSOURCE-824](https://folio-org.atlassian.net/browse/MODSOURCE-824) Endpoint /batch/parsed-records/fetch does not return deleted records

## 2024-11-26 5.9.2
* [MODSOURCE-820](https://folio-org.atlassian.net/browse/MODSOURCE-820) A job with multiple authority match profiles does not work as expected
* [MODSOURCE-816](https://folio-org.atlassian.net/browse/MODSOURCE-816) [RRT] Optimize execution plan for streaming SQL
Expand All @@ -20,6 +23,8 @@
* [MODSOURMAN-1203](https://folio-org.atlassian.net/browse/MODSOURMAN-1203) Add validation on MARC_BIB record save
* [MODSOURCE-796](https://folio-org.atlassian.net/browse/MODSOURCE-796) Fix inconsistencies in permission namings
* [MODSOURCE-809](https://folio-org.atlassian.net/browse/MODSOURCE-809) mod-source-record-storage Ramsons 2024 R2 - RMB v35.3.x update
* [MODSOURCE-787](https://folio-org.atlassian.net/browse/MODSOURCE-787) Extend MARC-MARC search query to account for comparison part
* [MODSOURCE-791](https://folio-org.atlassian.net/browse/MODSOURCE-791) Fix Timeout exception during postSourceStorageStreamMarcRecordIdentifiers

## 2024-03-20 5.8.0
* [MODSOURCE-733](https://issues.folio.org/browse/MODSOURCE-733) Reduce Memory Allocation of Strings
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,10 +70,11 @@ public interface RecordDao {
* @param externalIds list of ids
* @param idType external id type on which source record will be searched
* @param recordType record type
* @param includeDeleted searching by deleted records
* @param tenantId tenant id
* @return {@link Future} of {@link StrippedParsedRecordCollection}
*/
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, String tenantId);
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, Boolean includeDeleted, String tenantId);

/**
* Searches for {@link Record} by {@link MatchField} with offset and limit
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,8 @@ public Future<StrippedParsedRecordCollection> fetchStrippedParsedRecords(FetchPa
}

var recordType = toRecordType(fetchRequest.getRecordType().name());
return recordDao.getStrippedParsedRecords(ids, idType, recordType, tenantId)
var includeDeleted = fetchRequest.getIncludeDeleted();
return recordDao.getStrippedParsedRecords(ids, idType, recordType, includeDeleted, tenantId)
.onComplete(records -> filterFieldsByDataRange(records, fetchRequest))
.onFailure(ex -> {
LOG.warn("fetchParsedRecords:: Failed to fetch parsed records. {}", ex.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,46 @@ public void shouldPostSourceStorageBatchEdifactRecords(TestContext testContext)
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
public void shouldPostFetchParsedRecordsBatchWithDeletedWhenIncludeDeleteTrue(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
Expand All @@ -243,7 +281,8 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
Expand All @@ -257,6 +296,147 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
async.complete();
}


@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteFalse(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(false);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteNotExists(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}


@Test
public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
Expand All @@ -282,6 +462,38 @@ public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()))
.body("totalRecords", is(externalIds.size()));
async.complete();
}

@Test
public void shouldFailWhenPostSourceStorageBatchRecordsWithMultipleSnapshots(TestContext testContext) {
Async async = testContext.async();
Expand Down
Loading

0 comments on commit 4e5baa9

Please sign in to comment.