Skip to content

Commit

Permalink
Merge branch 'master' into MODSOURCE-817
Browse files Browse the repository at this point in the history
  • Loading branch information
@mukhiddin-yusuf
mukhiddin-yusuf authored Nov 26, 2024
2 parents 80c6847 + e85657d commit d432c83
Show file tree
Hide file tree
Showing 7 changed files with 457 additions and 96 deletions.
3 changes: 2 additions & 1 deletion NEWS.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
## 2024-XX-XX 5.10.0-SNAPSHOT
## 2025-XX-XX 5.10.0-SNAPSHOT
* [MODSOURCE-817](https://folio-org.atlassian.net/browse/MODSOURCE-817) Fix data consistency in handling and updating Marc Bib records for links.instance-authority event
* [MODSOURCE-816](https://folio-org.atlassian.net/browse/MODSOURCE-816) [RRT] Optimize execution plan for streaming SQL
* [MODSOURCE-824](https://folio-org.atlassian.net/browse/MODSOURCE-824) Endpoint /batch/parsed-records/fetch does not return deleted records

## 2024-10-28 5.9.0
* [MODSOURCE-767](https://folio-org.atlassian.net/browse/MODSOURCE-767) Single record overlay creates duplicate OCLC#/035
Expand Down
3 changes: 2 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDao.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,11 @@ public interface RecordDao {
* @param externalIds list of ids
* @param idType external id type on which source record will be searched
* @param recordType record type
* @param includeDeleted searching by deleted records
* @param tenantId tenant id
* @return {@link Future} of {@link StrippedParsedRecordCollection}
*/
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, String tenantId);
Future<StrippedParsedRecordCollection> getStrippedParsedRecords(List<String> externalIds, IdType idType, RecordType recordType, Boolean includeDeleted, String tenantId);

/**
* Searches for {@link Record} by {@link MatchField} with offset and limit
Expand Down
189 changes: 99 additions & 90 deletions mod-source-record-storage-server/src/main/java/org/folio/dao/RecordDaoImpl.java
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion mod-source-record-storage-server/src/main/java/org/folio/services/RecordServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,8 @@ public Future<StrippedParsedRecordCollection> fetchStrippedParsedRecords(FetchPa
}

var recordType = toRecordType(fetchRequest.getRecordType().name());
return recordDao.getStrippedParsedRecords(ids, idType, recordType, tenantId)
var includeDeleted = fetchRequest.getIncludeDeleted();
return recordDao.getStrippedParsedRecords(ids, idType, recordType, includeDeleted, tenantId)
.onComplete(records -> filterFieldsByDataRange(records, fetchRequest))
.onFailure(ex -> {
LOG.warn("fetchParsedRecords:: Failed to fetch parsed records. {}", ex.getMessage());
Expand Down
216 changes: 214 additions & 2 deletions ...ce-record-storage-server/src/test/java/org/folio/rest/impl/SourceStorageBatchApiTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,8 +227,46 @@ public void shouldPostSourceStorageBatchEdifactRecords(TestContext testContext)
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
public void shouldPostFetchParsedRecordsBatchWithDeletedWhenIncludeDeleteTrue(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
Expand All @@ -243,7 +281,8 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
Expand All @@ -257,6 +296,147 @@ public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
async.complete();
}


@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteFalse(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(false);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatchWithActualWhenIncludeDeleteNotExists(TestContext testContext) {
Async async = testContext.async();

Record record_1 = new Record()
.withId(FIRST_UUID)
.withSnapshotId(snapshot_1.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withMatchedId(FIRST_UUID)
.withOrder(0)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));
Record record_2 = new Record()
.withId(SECOND_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecord)
.withMatchedId(SECOND_UUID)
.withOrder(11)
.withState(Record.State.DELETED)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(MARC_RECORD_HRID));
Record record_3 = new Record()
.withId(THIRD_UUID)
.withSnapshotId(snapshot_2.getJobExecutionId())
.withRecordType(Record.RecordType.MARC_BIB)
.withRawRecord(rawRecord)
.withParsedRecord(marcRecordWithHrId)
.withErrorRecord(errorRecord)
.withMatchedId(THIRD_UUID)
.withState(Record.State.ACTUAL)
.withExternalIdsHolder(new ExternalIdsHolder()
.withInstanceId(UUID.randomUUID().toString())
.withInstanceHrid(HRID));

var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList());

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()-1))
.body("totalRecords", is(externalIds.size()-1));
async.complete();
}


@Test
public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
Expand All @@ -282,6 +462,38 @@ public void shouldPostFetchEmptyParsedRecordsBatch(TestContext testContext) {
async.complete();
}

@Test
public void shouldPostFetchParsedRecordsBatch(TestContext testContext) {
Async async = testContext.async();
var externalIds = List.of(
record_1.getExternalIdsHolder().getInstanceId(),
record_2.getExternalIdsHolder().getInstanceId(),
record_3.getExternalIdsHolder().getInstanceId()
);
postSnapshots(testContext, snapshot_1, snapshot_2, snapshot_3);
postRecords(testContext, record_1, record_2, record_3);

Conditions conditions = new Conditions()
.withIdType(IdType.INSTANCE.name())
.withIds(externalIds);
FetchParsedRecordsBatchRequest batchRequest = new FetchParsedRecordsBatchRequest()
.withRecordType(FetchParsedRecordsBatchRequest.RecordType.MARC_BIB)
.withConditions(conditions)
.withData(emptyList())
.withIncludeDeleted(true);

RestAssured.given()
.spec(spec)
.body(batchRequest)
.when()
.post(SOURCE_STORAGE_BATCH_FETCH_PARSED_RECORDS_PATH)
.then()
.statusCode(HttpStatus.SC_OK)
.body("records.size()", is(externalIds.size()))
.body("totalRecords", is(externalIds.size()));
async.complete();
}

@Test
public void shouldFailWhenPostSourceStorageBatchRecordsWithMultipleSnapshots(TestContext testContext) {
Async async = testContext.async();
Expand Down
Loading

0 comments on commit d432c83

Please sign in to comment.