Feature/performance improvements (#71)

* Batch copy * Testing * Assume atomic action * Trivy * Remove unused libraries * Leave refs null * Code review * Test fix * separate specimen only endpoint * separate creation of handles for different object types * separate creation of handles for different object types * Test fix * Test * Remove upsert endpoint * Code review * trivy * fix docker * Update trivy
DiSSCo · Dec 4, 2023 · 9466907 · 9466907
1 parent f31899b
commit 9466907
Show file tree

Hide file tree

Showing 16 changed files with 349 additions and 683 deletions.
diff --git a/.github/workflows/.trivyignore b/.github/workflows/.trivyignore
@@ -1,8 +1,4 @@
-# Date: Nov 14, 2023
-# Notes: Issue with openssl. May need to upgrade alpine image.
-CVE-2023-5363
-CVE-2023-5678
-
-# Date: Nov 21, 2023
-# Notes: Issue with netty. Spring boot needs to update
-CVE-2023-34062
+# Date: Dec 1, 2023
+# Notes: Issue with logback. Spring boot needs to update its version
+# Currently forcing logback version
+# CVE-2023-6378
diff --git a/Dockerfile b/Dockerfile
@@ -16,4 +16,4 @@ RUN true
 COPY --chown=java:java --from=builder application/application/ ./
 USER 1000
 
-ENTRYPOINT ["java", "org.springframework.boot.loader.JarLauncher"]
+ENTRYPOINT ["java", "org.springframework.boot.loader.launch.JarLauncher"]
diff --git a/pom.xml b/pom.xml
@@ -6,7 +6,7 @@
     <artifactId>spring-boot-starter-parent</artifactId>
     <groupId>org.springframework.boot</groupId>
     <relativePath/>
-    <version>3.1.5</version> <!-- lookup parent from repository -->
+    <version>3.2.0</version> <!-- lookup parent from repository -->
   </parent>
   <artifactId>handle-manager</artifactId>
   <version>0.0.1-SNAPSHOT</version>
@@ -19,14 +19,13 @@
     <java.version>17</java.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <testcontainers.version>1.17.6</testcontainers.version>
-    <victools.version>4.28.0</victools.version>
     <apache-commons.version>4.3</apache-commons.version>
     <mockito-inline.version>5.2.0</mockito-inline.version>
     <okhttp3.version>4.10.0</okhttp3.version>
+    <logback.version>1.4.12</logback.version>
     <sonar.coverage.jacoco.xmlReportPaths>
       ../app-it/target/site/jacoco-aggregate/jacoco.xml
     </sonar.coverage.jacoco.xmlReportPaths>
-    <snakeyaml.version>2.0</snakeyaml.version>
     <sonar.host.url>https://sonarcloud.io</sonar.host.url>
     <sonar.organization>dissco</sonar.organization>
     <version.victools>4.28.0</version.victools>

diff --git a/src/main/java/eu/dissco/core/handlemanager/controller/PidController.java b/src/main/java/eu/dissco/core/handlemanager/controller/PidController.java
@@ -1,20 +1,15 @@
 package eu.dissco.core.handlemanager.controller;
 
 
-import static eu.dissco.core.handlemanager.domain.FdoProfile.PRIMARY_SPECIMEN_OBJECT_ID;
-import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_ATTRIBUTES;
 import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_DATA;
 import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_ID;
-import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_TYPE;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperRead;
 import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperReadSingle;
 import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperWrite;
 import eu.dissco.core.handlemanager.domain.requests.RollbackRequest;
 import eu.dissco.core.handlemanager.domain.requests.validation.JsonSchemaValidator;
-import eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType;
 import eu.dissco.core.handlemanager.exceptions.InvalidRequestException;
 import eu.dissco.core.handlemanager.exceptions.PidCreationException;
 import eu.dissco.core.handlemanager.exceptions.PidResolutionException;
@@ -166,28 +161,6 @@ public ResponseEntity<JsonApiWrapperWrite> updateRecords(@RequestBody List<JsonN
     return ResponseEntity.status(HttpStatus.OK).body(service.updateRecords(requests, true));
   }
 
-  // Upsert
-  @Operation(summary = "Create a PID Record; if it already exists, update contents. DigitalSpecimens only.")
-  @PatchMapping(value = "/upsert")
-  public ResponseEntity<JsonApiWrapperWrite> upsertRecord(@RequestBody List<JsonNode> requests,
-      Authentication authentication)
-      throws InvalidRequestException, UnprocessableEntityException, PidResolutionException, JsonProcessingException {
-    log.info("Validating upsert request from user {}", authentication.getName());
-    for (var request : requests) {
-      schemaValidator.validatePostRequest(request);
-      if (!request.get(NODE_DATA).get(NODE_TYPE).asText()
-          .equals(ObjectType.DIGITAL_SPECIMEN.toString())) {
-        log.error(
-            "Attempting to upsert invalid type of PID record. Currently only upsert for Digital Specimen is supported.");
-        throw new InvalidRequestException(
-            "Invalid type. Upsert endpoint only available for type DigitalSpecimen");
-      }
-    }
-    var ids = getPhysicalSpecimenIds(requests);
-    log.info("Received valid upsert request for {} physical specimens: {}... ", requests.size(),
-        ids);
-    return ResponseEntity.ok(service.upsertDigitalSpecimens(requests));
-  }
 
   @Operation(summary = "Archive given record")
   @PutMapping(value = "/{prefix}/{suffix}")
@@ -268,8 +241,4 @@ public ResponseEntity<JsonApiWrapperWrite> archiveRecords(@RequestBody List<Json
     return ResponseEntity.status(HttpStatus.OK).body(service.archiveRecordBatch(requests));
   }
 
-  private List<String> getPhysicalSpecimenIds(List<JsonNode> requests) {
-    return requests.stream().map(r -> r.get(NODE_DATA).get(NODE_ATTRIBUTES)
-        .get(PRIMARY_SPECIMEN_OBJECT_ID.get()).asText()).limit(LOG_LIMIT).toList();
-  }
 }
diff --git a/src/main/java/eu/dissco/core/handlemanager/domain/requests/UpsertDigitalSpecimen.java b/src/main/java/eu/dissco/core/handlemanager/domain/requests/UpsertDigitalSpecimen.java
diff --git a/src/main/java/eu/dissco/core/handlemanager/exceptions/UnprocessableEntityException.java b/src/main/java/eu/dissco/core/handlemanager/exceptions/UnprocessableEntityException.java
@@ -1,8 +1,8 @@
 package eu.dissco.core.handlemanager.exceptions;
 
-public class UnprocessableEntityException extends Exception {
+public class UnprocessableEntityException extends RuntimeException {
 
-  public UnprocessableEntityException(String s){
+  public UnprocessableEntityException(String s) {
     super(s);
   }
 

diff --git a/src/main/java/eu/dissco/core/handlemanager/service/DoiService.java b/src/main/java/eu/dissco/core/handlemanager/service/DoiService.java
@@ -4,28 +4,23 @@
 import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_ATTRIBUTES;
 import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_DATA;
 import static eu.dissco.core.handlemanager.domain.JsonApiFields.NODE_TYPE;
+import static eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType.DIGITAL_SPECIMEN;
+import static eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType.MEDIA_OBJECT;
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import eu.dissco.core.handlemanager.Profiles;
 import eu.dissco.core.handlemanager.domain.jsonapi.JsonApiWrapperWrite;
 import eu.dissco.core.handlemanager.domain.repsitoryobjects.HandleAttribute;
-import eu.dissco.core.handlemanager.domain.requests.objects.DigitalSpecimenRequest;
-import eu.dissco.core.handlemanager.domain.requests.objects.MediaObjectRequest;
-import eu.dissco.core.handlemanager.domain.requests.vocabulary.specimen.ObjectType;
 import eu.dissco.core.handlemanager.exceptions.InvalidRequestException;
 import eu.dissco.core.handlemanager.exceptions.PidCreationException;
 import eu.dissco.core.handlemanager.exceptions.PidResolutionException;
 import eu.dissco.core.handlemanager.exceptions.UnprocessableEntityException;
 import eu.dissco.core.handlemanager.properties.ProfileProperties;
 import eu.dissco.core.handlemanager.repository.PidRepository;
-import java.nio.charset.StandardCharsets;
 import java.time.Instant;
-import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Profile;
@@ -46,73 +41,43 @@ public DoiService(PidRepository pidRepository,
   private static final String TYPE_ERROR_MESSAGE = "Error creating DOI for object of Type %s. Only Digital Specimens and Media Objects use DOIs.";
 
   @Override
-  public JsonApiWrapperWrite createRecords(
-      List<JsonNode> requests)
-      throws PidResolutionException, InvalidRequestException, PidCreationException {
-
-    var handles = hf.genHandleList(requests.size());
-    var handleIterator = handles.iterator();
-    List<DigitalSpecimenRequest> digitalSpecimenList = new ArrayList<>();
-
-    List<HandleAttribute> handleAttributes = new ArrayList<>();
-    Map<String, ObjectType> recordTypes = new HashMap<>();
-
-    for (var request : requests) {
-      var dataNode = request.get(NODE_DATA);
-      var thisHandle = handleIterator.next();
-      ObjectType type = ObjectType.fromString(dataNode.get(NODE_TYPE).asText());
-      recordTypes.put(new String(thisHandle, StandardCharsets.UTF_8), type);
-      try {
-        switch (type) {
-          case DIGITAL_SPECIMEN -> {
-            var requestObject = mapper.treeToValue(dataNode.get(NODE_ATTRIBUTES),
-                DigitalSpecimenRequest.class);
-            handleAttributes.addAll(
-                fdoRecordService.prepareDigitalSpecimenRecordAttributes(requestObject,
-                    thisHandle, type));
-            digitalSpecimenList.add(requestObject);
-          }
-          case MEDIA_OBJECT -> {
-            var requestObject = mapper.treeToValue(dataNode.get(NODE_ATTRIBUTES),
-                MediaObjectRequest.class);
-            handleAttributes.addAll(
-                fdoRecordService.prepareMediaObjectAttributes(requestObject, thisHandle,
-                    type));
-          }
-          default -> throw new InvalidRequestException(String.format(
-              TYPE_ERROR_MESSAGE, type));
-        }
-      } catch (JsonProcessingException | UnprocessableEntityException e) {
-        throw new InvalidRequestException(
-            "An error has occurred parsing a record in request. More information: "
-                + e.getMessage());
+  public JsonApiWrapperWrite createRecords(List<JsonNode> requests)
+      throws InvalidRequestException, PidCreationException {
+    var handles = hf.genHandleList(requests.size()).iterator();
+    var requestAttributes = requests.stream()
+        .map(request -> request.get(NODE_DATA).get(NODE_ATTRIBUTES)).toList();
+    var type = getObjectType(requests);
+    List<HandleAttribute> handleAttributes;
+    try {
+      switch (type) {
+        case DIGITAL_SPECIMEN ->
+            handleAttributes = createDigitalSpecimen(requestAttributes, handles);
+        case MEDIA_OBJECT -> handleAttributes = createMediaObject(requestAttributes, handles);
+        default -> throw new UnsupportedOperationException(
+            type + " is not an appropriate Type for DOI endpoint.");
       }
+    } catch (JsonProcessingException | PidResolutionException e) {
+      throw new InvalidRequestException(
+          "An error has occurred parsing a record in request. More information: " + e.getMessage());
     }
-
-    validateDigitalSpecimens(digitalSpecimenList);
-
-    log.info("Persisting new DOIs to db");
-    var recordTimestamp = Instant.now().getEpochSecond();
-
-    pidRepository.postAttributesToDb(recordTimestamp, handleAttributes);
-
-    return new JsonApiWrapperWrite(formatCreateRecords(handleAttributes, recordTypes));
+    log.info("Persisting new dois to db");
+    pidRepository.postAttributesToDb(Instant.now().getEpochSecond(), handleAttributes);
+    return new JsonApiWrapperWrite(formatCreateRecords(handleAttributes, type));
   }
 
   @Override
   public JsonApiWrapperWrite updateRecords(List<JsonNode> requests, boolean incrementVersion)
       throws InvalidRequestException, PidResolutionException, UnprocessableEntityException {
     var types = requests.stream()
         .map(request -> request.get(NODE_DATA).get(NODE_TYPE).asText())
-        .filter(type -> !type.equals(ObjectType.MEDIA_OBJECT.toString())
-            || !type.equals(ObjectType.DIGITAL_SPECIMEN.toString()))
+        .filter(type -> !type.equals(MEDIA_OBJECT.toString())
+            || !type.equals(DIGITAL_SPECIMEN.toString()))
         .collect(Collectors.toSet());
 
     if (!types.isEmpty()) {
       throw new InvalidRequestException(String.format(TYPE_ERROR_MESSAGE, types));
     }
     return super.updateRecords(requests, incrementVersion);
-
   }