Skip to content

Commit

Permalink
Remove upsert endpoint
Browse files Browse the repository at this point in the history
  • Loading branch information
@southeo
southeo committed Nov 23, 2023
1 parent 80aa059 commit 969e6e5
Show file tree
Hide file tree
Showing 5 changed files with 0 additions and 358 deletions.
44 changes: 0 additions & 44 deletions src/main/java/eu/dissco/core/handlemanager/controller/PidController.java
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,15 @@
package eu.dissco.core.handlemanager.controller;


import static eu.dissco.core.handlemanager.domain.FdoProfile.PRIMARY_SPECIMEN_OBJECT_ID;
import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_ATTRIBUTES;
import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_DATA;
import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_ID;
import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_TYPE;

import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperRead;
import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperReadSingle;
import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperWrite;
import eu.dissco.core.handlemanager.domain.requests.RollbackRequest;
import eu.dissco.core.handlemanager.domain.requests.validation.JsonSchemaValidator;
import eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType;
import eu.dissco.core.handlemanager.exceptions.InvalidRequestException;
import eu.dissco.core.handlemanager.exceptions.PidCreationException;
import eu.dissco.core.handlemanager.exceptions.PidResolutionException;
Expand All @@ -24,8 +19,6 @@
import io.swagger.v3.oas.annotations.Operation;
import jakarta.servlet.http.HttpServletRequest;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand Down Expand Up @@ -168,39 +161,6 @@ public ResponseEntity<JsonApiWrapperWrite> updateRecords(@RequestBody List<JsonN
return ResponseEntity.status(HttpStatus.OK).body(service.updateRecords(requests, true));
}

// Upsert

/**
* @deprecated (Use the POST and PATCH batch endpoints instead)
*/
@Operation(summary = "Create a PID Record; if it already exists, update contents. DigitalSpecimens only.")
@PatchMapping(value = "/upsert")
@Deprecated(forRemoval = true)
public ResponseEntity<JsonApiWrapperWrite> upsertRecord(@RequestBody List<JsonNode> requests,
Authentication authentication)
throws InvalidRequestException, UnprocessableEntityException, PidResolutionException, JsonProcessingException {
log.info("Validating upsert request from user {}", authentication.getName());
var start = Instant.now();
for (var request : requests) {
schemaValidator.validatePostRequest(request);
if (!request.get(NODE_DATA).get(NODE_TYPE).asText()
.equals(ObjectType.DIGITAL_SPECIMEN.toString())) {
log.error(
"Attempting to upsert invalid type of PID record. Currently only upsert for Digital Specimen is supported.");
throw new InvalidRequestException(
"Invalid type. Upsert endpoint only available for type DigitalSpecimen");
}
}
var ids = getPhysicalSpecimenIds(requests);
log.info("Received valid upsert request for {} physical specimens: {}... ", requests.size(),
ids);
var result = service.upsertDigitalSpecimens(requests);
var elapsed = Duration.between(start, Instant.now()).toNanos() / (Math.pow(10, 9));
var rate = requests.size() / elapsed;
log.info("{} seconds elapsed. Rate: {} specimens / second", elapsed, rate);
log.info("****,{},{},{}", requests.size(), elapsed, rate);
return ResponseEntity.ok(result);
}

@Operation(summary = "Archive given record")
@PutMapping(value = "/{prefix}/{suffix}")
Expand Down Expand Up @@ -281,8 +241,4 @@ public ResponseEntity<JsonApiWrapperWrite> archiveRecords(@RequestBody List<Json
return ResponseEntity.status(HttpStatus.OK).body(service.archiveRecordBatch(requests));
}

private List<String> getPhysicalSpecimenIds(List<JsonNode> requests) {
return requests.stream().map(r -> r.get(NODE_DATA).get(NODE_ATTRIBUTES)
.get(PRIMARY_SPECIMEN_OBJECT_ID.get()).asText()).limit(LOG_LIMIT).toList();
}
}
11 changes: 0 additions & 11 deletions src/main/java/eu/dissco/core/handlemanager/domain/requests/UpsertDigitalSpecimen.java
View file

This file was deleted.

149 changes: 0 additions & 149 deletions src/main/java/eu/dissco/core/handlemanager/service/PidService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperReadSingle;
import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperWrite;
import eu.dissco.core.handlemanager.domain.repsitoryobjects.HandleAttribute;
import eu.dissco.core.handlemanager.domain.requests.UpsertDigitalSpecimen;
import eu.dissco.core.handlemanager.domain.requests.objects.DigitalSpecimenRequest;
import eu.dissco.core.handlemanager.domain.requests.objects.MediaObjectRequest;
import eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType;
Expand All @@ -46,7 +45,6 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;

Expand Down Expand Up @@ -172,22 +170,6 @@ private List<JsonApiDataLinks> formatCreateRecordsDefault(
return dataLinksList;
}

private List<JsonApiDataLinks> formatUpsertResponse(List<HandleAttribute> records) {
List<JsonApiDataLinks> dataLinksList = new ArrayList<>();
for (var row : records) {
if (row.getType().equals(PRIMARY_SPECIMEN_OBJECT_ID.get())) {
String h = new String(row.getHandle(), StandardCharsets.UTF_8);
String pidLink = profileProperties.getDomain() + h;
var node = mapper.createObjectNode();
node.put(PRIMARY_SPECIMEN_OBJECT_ID.get(),
new String(row.getData(), StandardCharsets.UTF_8));
dataLinksList.add(
new JsonApiDataLinks(h, DIGITAL_SPECIMEN.toString(), node, new JsonApiLinks(pidLink)));
}
}
return dataLinksList;
}

private JsonApiWrapperWrite formatArchives(List<List<HandleAttribute>> archiveRecords) {
List<JsonApiDataLinks> dataList = new ArrayList<>();
for (var archiveRecord : archiveRecords) {
Expand Down Expand Up @@ -346,137 +328,6 @@ protected <T extends DigitalSpecimenRequest> void verifyNoInternalDuplicatePhysi
}
}

// Upsert
public JsonApiWrapperWrite upsertDigitalSpecimens(List<JsonNode> requests)
throws JsonProcessingException, UnprocessableEntityException, PidResolutionException,
InvalidRequestException {
var digitalSpecimenRequests = jsonNodeToDigitalSpecimenRequest(requests);

var physicalIds = getPhysicalIdsFromRequests(digitalSpecimenRequests);
var physicalIdsBytes = getPhysIdBytes(physicalIds);
var upsertRequests = getRegisteredSpecimensUpsert(digitalSpecimenRequests, physicalIdsBytes);
var upsertAttributes = prepareUpsertAttributes(upsertRequests);
logUpdates(upsertRequests);

var createRequests = getCreateRequests(upsertRequests, digitalSpecimenRequests);
var newHandles = hf.genHandleList(createRequests.size());
if (!newHandles.isEmpty()) {
log.info("Successfully minted {} new identifiers(s)", newHandles.size());
}
var createAttributes = getCreateAttributes(createRequests, newHandles);

var allRequests = Stream.concat(
createRequests.stream(),
upsertRequests.stream().map(UpsertDigitalSpecimen::request)).toList();
verifyNoInternalDuplicatePhysicalSpecimenObjectId(allRequests, physicalIds);

var recordTimestamp = Instant.now().getEpochSecond();

log.info("Persisting upserts to db.");
pidRepository.postAndUpdateHandles(recordTimestamp, createAttributes, upsertAttributes);

var concatAttributes = concatHandleAttributes(createAttributes, upsertAttributes);

return new JsonApiWrapperWrite(
formatUpsertResponse(concatAttributes));
}

private List<DigitalSpecimenRequest> jsonNodeToDigitalSpecimenRequest(List<JsonNode> requests)
throws JsonProcessingException {
ArrayList<DigitalSpecimenRequest> digitalSpecimenRequests = new ArrayList<>();
for (var request : requests) {
digitalSpecimenRequests.add(mapper.treeToValue(request.get(NODE_DATA).get(NODE_ATTRIBUTES),
DigitalSpecimenRequest.class));
}
return digitalSpecimenRequests;
}

private List<UpsertDigitalSpecimen> getRegisteredSpecimensUpsert(
List<DigitalSpecimenRequest> requests, List<byte[]> physicalIds) {
var registeredSpecimensHandleAttributes = new HashSet<>(
pidRepository.searchByNormalisedPhysicalIdentifier(physicalIds));
if (registeredSpecimensHandleAttributes.isEmpty()) {
return new ArrayList<>();
}

ArrayList<UpsertDigitalSpecimen> upsertDigitalSpecimen = new ArrayList<>();
for (var row : registeredSpecimensHandleAttributes) {
var targetPhysId = new String(row.getData(), StandardCharsets.UTF_8);
var targetRequest = getRequestFromPhysicalId(requests, targetPhysId);
requests.remove(targetRequest);
upsertDigitalSpecimen.add(new UpsertDigitalSpecimen(
new String(row.getHandle(), StandardCharsets.UTF_8),
targetPhysId,
targetRequest
));
}
return upsertDigitalSpecimen;
}

private void logUpdates(List<UpsertDigitalSpecimen> upsertRequests) {
var registeredHandles = upsertRequests.stream().map(UpsertDigitalSpecimen::handle).toList();
if (!registeredHandles.isEmpty()) {
log.debug("Some specimens already have identifiers. Updating the following PID Records {}",
registeredHandles);
}
}

private DigitalSpecimenRequest getRequestFromPhysicalId(List<DigitalSpecimenRequest> requests,
String physicalId) {
for (var request : requests) {
if (physicalId.equals(request.getNormalisedPrimarySpecimenObjectId())) {
return request;
}
}
throw new IllegalStateException("Physical Identifier not found");
}

private List<DigitalSpecimenRequest> getCreateRequests(List<UpsertDigitalSpecimen> upsertRequests,
List<DigitalSpecimenRequest> digitalSpecimenRequests) {
var upsertRequestsSet = upsertRequests
.stream()
.map(UpsertDigitalSpecimen::request).collect(Collectors.toSet());
return digitalSpecimenRequests
.stream()
.filter(r -> (!upsertRequestsSet.contains(r)))
.toList();
}

private List<HandleAttribute> getCreateAttributes(
List<DigitalSpecimenRequest> digitalSpecimenRequests, List<byte[]> newHandles)
throws UnprocessableEntityException, PidResolutionException, InvalidRequestException {
var handles = new ArrayList<>(newHandles);
List<HandleAttribute> handleAttributes = new ArrayList<>();
for (var digitalSpecimenRequest : digitalSpecimenRequests) {
handleAttributes.addAll(
fdoRecordService.prepareDigitalSpecimenRecordAttributes(digitalSpecimenRequest,
handles.remove(0)));
}
return handleAttributes;
}

private List<HandleAttribute> concatHandleAttributes(List<HandleAttribute> createAttributes,
List<List<HandleAttribute>> upsertAttributes) {
List<HandleAttribute> upsertListFlat = new ArrayList<>();
for (var upsertRecord : upsertAttributes) {
upsertListFlat.addAll(upsertRecord);
}
return Stream.concat(createAttributes.stream(), upsertListFlat.stream()).toList();
}

private List<List<HandleAttribute>> prepareUpsertAttributes(
List<UpsertDigitalSpecimen> upsertDigitalSpecimens)
throws InvalidRequestException, UnprocessableEntityException, PidResolutionException {
List<List<HandleAttribute>> upsertAttributes = new ArrayList<>();
for (var upsertRequest : upsertDigitalSpecimens) {
ArrayList<HandleAttribute> upsertAttributeSingleSpecimen = new ArrayList<>(fdoRecordService
.prepareUpdateAttributes(upsertRequest.handle().getBytes(StandardCharsets.UTF_8),
mapper.valueToTree(upsertRequest.request()), DIGITAL_SPECIMEN));
upsertAttributes.add(upsertAttributeSingleSpecimen);
}
return upsertAttributes;
}

// Update
public JsonApiWrapperWrite updateRecords(List<JsonNode> requests,
boolean incrementVersion)
Expand Down
25 changes: 0 additions & 25 deletions src/test/java/eu/dissco/core/handlemanager/controller/PidControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -583,31 +583,6 @@ void testRollbackHandlesBadRequest() {
() -> controller.rollbackHandleCreation(request, authentication));
}


@Test
void testUpsert() throws Exception {
// Given
var request = genCreateRecordRequest(givenDigitalSpecimenRequestObjectNullOptionals(),
RECORD_TYPE_DS);

// When
var response = controller.upsertRecord(List.of(request), authentication);

// Then
assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK);
}

@Test
void testUpsertBadType() {
// Given
var request = genCreateRecordRequest(givenDigitalSpecimenRequestObjectNullOptionals(),
RECORD_TYPE_MAPPING);

// Then
assertThrows(InvalidRequestException.class, () -> controller.upsertRecord(List.of(request),
authentication));
}

@Test
void testArchiveRecord() throws Exception {
// Given
Expand Down
Loading

0 comments on commit 969e6e5

Please sign in to comment.