Merge pull request #60 from kbss-cvut/184-support-excel-import

184 support excel import
kbss-cvut · Jul 21, 2024 · 86e9419 · 86e9419
2 parents 9326c05 + 6fff1db
commit 86e9419
Show file tree

Hide file tree

Showing 5 changed files with 156 additions and 17 deletions.
diff --git a/src/main/java/cz/cvut/kbss/study/config/ServiceConfig.java b/src/main/java/cz/cvut/kbss/study/config/ServiceConfig.java
@@ -7,6 +7,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
+import org.springframework.http.converter.FormHttpMessageConverter;
 import org.springframework.http.converter.ResourceHttpMessageConverter;
 import org.springframework.http.converter.StringHttpMessageConverter;
 import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter;
@@ -40,8 +41,12 @@ public RestTemplate restTemplate(ObjectMapper objectMapper) {
         final MappingJackson2HttpMessageConverter jacksonConverter = new MappingJackson2HttpMessageConverter();
         jacksonConverter.setObjectMapper(objectMapper);
         final StringHttpMessageConverter stringConverter = new StringHttpMessageConverter(StandardCharsets.UTF_8);
-        restTemplate.setMessageConverters(
-                Arrays.asList(stringConverter, jacksonConverter, new ResourceHttpMessageConverter()));
+        restTemplate.setMessageConverters(Arrays.asList(
+            stringConverter,
+            jacksonConverter,
+            new ResourceHttpMessageConverter(),
+            new FormHttpMessageConverter()
+        ));
         return restTemplate;
     }
 }
diff --git a/src/main/java/cz/cvut/kbss/study/rest/PatientRecordController.java b/src/main/java/cz/cvut/kbss/study/rest/PatientRecordController.java
@@ -1,5 +1,7 @@
 package cz.cvut.kbss.study.rest;
 
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import cz.cvut.kbss.study.dto.PatientRecordDto;
 import cz.cvut.kbss.study.dto.RecordImportResult;
 import cz.cvut.kbss.study.exception.NotFoundException;
@@ -20,15 +22,19 @@
 import jakarta.servlet.http.HttpServletRequest;
 import jakarta.servlet.http.HttpServletResponse;
 import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.core.ParameterizedTypeReference;
 import org.springframework.core.io.InputStreamResource;
 import org.springframework.data.domain.Page;
 import org.springframework.http.*;
 import org.springframework.security.access.prepost.PreAuthorize;
+import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
 import org.springframework.web.bind.annotation.*;
 import org.springframework.web.client.RestTemplate;
+import org.springframework.web.multipart.MultipartFile;
 import org.springframework.web.util.UriComponentsBuilder;
 
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.*;
@@ -45,13 +51,17 @@ public class PatientRecordController extends BaseController {
     private final ExcelRecordConverter excelRecordConverter;
     private final RestTemplate restTemplate;
     private final ConfigReader configReader;
+    private ObjectMapper objectMapper;
 
-    public PatientRecordController(PatientRecordService recordService, ApplicationEventPublisher eventPublisher, ExcelRecordConverter excelRecordConverter, RestTemplate restTemplate, ConfigReader configReader) {
+    public PatientRecordController(PatientRecordService recordService, ApplicationEventPublisher eventPublisher,
+                                   ExcelRecordConverter excelRecordConverter, RestTemplate restTemplate,
+                                   ConfigReader configReader, ObjectMapper objectMapper)  {
         this.recordService = recordService;
         this.eventPublisher = eventPublisher;
         this.excelRecordConverter = excelRecordConverter;
         this.restTemplate = restTemplate;
         this.configReader = configReader;
+        this.objectMapper = objectMapper;
     }
 
     @PreAuthorize("hasRole('" + SecurityConstants.ROLE_ADMIN + "') or @securityUtils.isMemberOfInstitution(#institutionKey)")
@@ -152,9 +162,102 @@ public ResponseEntity<Void> createRecord(@RequestBody PatientRecord record) {
         return new ResponseEntity<>(headers, HttpStatus.CREATED);
     }
 
-    @PostMapping(value = "/import", consumes = MediaType.APPLICATION_JSON_VALUE)
-    public RecordImportResult importRecords(@RequestBody List<PatientRecord> records,
+    @PostMapping(value = "/import/json", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    public RecordImportResult importRecordsJson(@RequestPart("file") MultipartFile file,
                                             @RequestParam(name = "phase", required = false) String phase) {
+
+        List<PatientRecord> records;
+
+        if(file.isEmpty())
+            throw new IllegalArgumentException("Cannot import records, missing input file");
+        try {
+            records =  objectMapper.readValue(file.getBytes(), new TypeReference<List<PatientRecord>>(){});
+        } catch (IOException e) {
+            throw new RuntimeException("Failed to parse JSON content", e);
+        }
+        return importRecords(records, phase);
+    }
+
+    @PostMapping(value = "/import/excel", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    public RecordImportResult importRecordsExcel(
+        @RequestPart("file") MultipartFile file,
+        @RequestParam(name = "phase", required = false) String phase) {
+
+        List<PatientRecord> records;
+
+        if(file.isEmpty())
+            throw new IllegalArgumentException("Cannot import records, missing input file");
+
+        String excelImportServiceUrl = configReader.getConfig(ConfigParam.EXCEL_IMPORT_SERVICE_URL);
+
+        if (excelImportServiceUrl == null)
+            throw new IllegalArgumentException("Cannot import XLS, excelImportServiceUrl is not configured");
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(org.springframework.http.MediaType.MULTIPART_FORM_DATA);
+
+        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+        body.add("files", file.getResource());
+
+        String request = UriComponentsBuilder.fromHttpUrl(excelImportServiceUrl)
+            .queryParam("datasetResource", "@%s".formatted(file.getOriginalFilename()))
+            .toUriString();
+
+        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
+
+        ResponseEntity<List<PatientRecord>> responseEntity = restTemplate.exchange(
+                URI.create(request),
+                HttpMethod.POST,
+                requestEntity,
+                new ParameterizedTypeReference<List<PatientRecord>>() {}
+        );
+        records = responseEntity.getBody();
+        return importRecords(records, phase);
+    }
+
+    @PostMapping(value = "/import/tsv", consumes = MediaType.MULTIPART_FORM_DATA_VALUE)
+    public RecordImportResult importRecordsTsv(
+        @RequestPart("file") MultipartFile file,
+        @RequestParam(name = "phase", required = false) String phase) {
+
+        List<PatientRecord> records;
+
+        if(file.isEmpty())
+            throw new IllegalArgumentException("Cannot import records, missing input file");
+
+        String excelImportServiceUrl = configReader.getConfig(ConfigParam.EXCEL_IMPORT_SERVICE_URL);
+
+        if (excelImportServiceUrl == null)
+            throw new IllegalArgumentException("Cannot import TSV, excelImportServiceUrl is not configured");
+
+        HttpHeaders headers = new HttpHeaders();
+        headers.setContentType(org.springframework.http.MediaType.MULTIPART_FORM_DATA);
+
+        MultiValueMap<String, Object> body = new LinkedMultiValueMap<>();
+        body.add("files", file.getResource());
+
+        String request = UriComponentsBuilder.fromHttpUrl(excelImportServiceUrl)
+            .queryParam("datasetResource", "@%s".formatted(file.getOriginalFilename()))
+            .toUriString();
+
+        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(body, headers);
+
+        ResponseEntity<byte[]> responseEntity = restTemplate.postForEntity(
+            URI.create(request),
+            requestEntity,
+            byte[].class
+        );
+
+        LOG.info("Import finished with status {}", responseEntity.getStatusCode());
+        if (responseEntity.getStatusCode() == HttpStatus.OK) {
+            byte[] responseBody = responseEntity.getBody();
+            LOG.debug("Response body length is {}", responseBody.length);
+        }
+
+        return new RecordImportResult();
+    }
+
+    public RecordImportResult importRecords(List<PatientRecord> records, String phase) {
         final RecordImportResult importResult;
         if (phase != null) {
             final RecordPhase targetPhase = RecordPhase.fromIriOrName(phase);
@@ -166,6 +269,7 @@ public RecordImportResult importRecords(@RequestBody List<PatientRecord> records
         return importResult;
     }
 
+
     @PutMapping(value = "/{key}", consumes = MediaType.APPLICATION_JSON_VALUE)
     @ResponseStatus(HttpStatus.NO_CONTENT)
     public void updateRecord(@PathVariable("key") String key, @RequestBody PatientRecord record) {

diff --git a/src/main/java/cz/cvut/kbss/study/service/PatientRecordService.java b/src/main/java/cz/cvut/kbss/study/service/PatientRecordService.java
@@ -72,6 +72,7 @@ public interface PatientRecordService extends BaseService<PatientRecord> {
      * current user is set as the record's author.
      *
      * @param records Records to import
+     * @param targetPhase Phase to be set to all imported records.
      * @return Instance representing the import result
      * @throws cz.cvut.kbss.study.exception.RecordAuthorNotFoundException Thrown when importing a record whose author
      *                                                                    does not exist in this application instance's

diff --git a/src/main/java/cz/cvut/kbss/study/util/ConfigParam.java b/src/main/java/cz/cvut/kbss/study/util/ConfigParam.java
@@ -10,6 +10,7 @@ public enum ConfigParam {
     FORM_GEN_SERVICE_URL("formGenServiceUrl"),
 
     ON_UPDATE_RECORD_SERVICE_URL("onRecordUpdateServiceUrl"),
+    EXCEL_IMPORT_SERVICE_URL("excelImportServiceUrl"),
 
     APP_CONTEXT("appContext"),
 

diff --git a/src/test/java/cz/cvut/kbss/study/rest/PatientRecordControllerTest.java b/src/test/java/cz/cvut/kbss/study/rest/PatientRecordControllerTest.java
@@ -1,6 +1,7 @@
 package cz.cvut.kbss.study.rest;
 
 import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import cz.cvut.kbss.study.dto.PatientRecordDto;
 import cz.cvut.kbss.study.dto.RecordImportResult;
 import cz.cvut.kbss.study.environment.generator.Generator;
@@ -23,6 +24,7 @@
 import org.mockito.ArgumentCaptor;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
+import org.mockito.Spy;
 import org.mockito.junit.jupiter.MockitoExtension;
 import org.springframework.context.ApplicationEventPublisher;
 import org.springframework.data.domain.Page;
@@ -32,6 +34,7 @@
 import org.springframework.data.domain.Sort;
 import org.springframework.http.HttpStatus;
 import org.springframework.http.MediaType;
+import org.springframework.mock.web.MockMultipartFile;
 import org.springframework.test.web.servlet.MvcResult;
 
 import java.time.LocalDate;
@@ -51,10 +54,7 @@
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.*;
 import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
 
 @ExtendWith(MockitoExtension.class)
@@ -66,6 +66,9 @@ public class PatientRecordControllerTest extends BaseControllerTestRunner {
     @Mock
     private ApplicationEventPublisher eventPublisherMock;
 
+    @Spy
+    private ObjectMapper objectMapper = Environment.getObjectMapper();
+
     @InjectMocks
     private PatientRecordController controller;
 
@@ -302,48 +305,73 @@ void exportRecordsExportsRecordsForProvidedInstitutionForSpecifiedPeriod() throw
                 Pageable.unpaged());
     }
 
+
     @Test
-    void importRecordsImportsSpecifiedRecordsAndReturnsImportResult() throws Exception {
+    void importRecordsJsonImportsSpecifiedRecordsAndReturnsImportResult() throws Exception {
         final List<PatientRecord> records =
                 List.of(Generator.generatePatientRecord(user), Generator.generatePatientRecord(user));
         final RecordImportResult importResult = new RecordImportResult(records.size());
         importResult.setImportedCount(records.size());
+
         when(patientRecordServiceMock.importRecords(anyList())).thenReturn(importResult);
 
+        MockMultipartFile file = new MockMultipartFile("file", "records.json",
+                MediaType.MULTIPART_FORM_DATA_VALUE, toJson(records).getBytes());
+
         final MvcResult mvcResult = mockMvc.perform(
-                post("/records/import").content(toJson(records)).contentType(MediaType.APPLICATION_JSON)).andReturn();
+                multipart("/records/import/json")
+                        .file(file)
+                        .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)
+        ).andReturn();
+
         final RecordImportResult result = readValue(mvcResult, RecordImportResult.class);
         assertEquals(importResult.getTotalCount(), result.getTotalCount());
         assertEquals(importResult.getImportedCount(), result.getImportedCount());
         assertThat(importResult.getErrors(), anyOf(nullValue(), empty()));
+
         @SuppressWarnings("unchecked")
         final ArgumentCaptor<List<PatientRecord>> captor = ArgumentCaptor.forClass(List.class);
         verify(patientRecordServiceMock).importRecords(captor.capture());
         assertEquals(records.size(), captor.getValue().size());
     }
 
+
     @Test
-    void importRecordsImportsSpecifiedRecordsWithSpecifiedPhaseAndReturnsImportResult() throws Exception {
+    void importRecordsJsonImportsSpecifiedRecordsWithSpecifiedPhaseAndReturnsImportResult() throws Exception {
         final List<PatientRecord> records =
                 List.of(Generator.generatePatientRecord(user), Generator.generatePatientRecord(user));
         final RecordImportResult importResult = new RecordImportResult(records.size());
         importResult.setImportedCount(records.size());
         final RecordPhase targetPhase = RecordPhase.values()[Generator.randomInt(0, RecordPhase.values().length)];
         when(patientRecordServiceMock.importRecords(anyList(), any(RecordPhase.class))).thenReturn(importResult);
 
-        mockMvc.perform(post("/records/import").content(toJson(records)).contentType(MediaType.APPLICATION_JSON)
-                                               .param("phase", targetPhase.getIri())).andExpect(status().isOk());
+        MockMultipartFile file = new MockMultipartFile("file", "records.json",
+            MediaType.MULTIPART_FORM_DATA_VALUE, toJson(records).getBytes());
+
+        mockMvc.perform(
+            multipart("/records/import/json")
+                .file(file)
+                .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)
+                .param("phase", targetPhase.getIri())
+        ).andExpect(status().isOk());
+
         verify(patientRecordServiceMock).importRecords(anyList(), eq(targetPhase));
     }
 
     @Test
-    void importRecordsReturnsConflictWhenServiceThrowsRecordAuthorNotFound() throws Exception {
+    void importRecordsJsonReturnsConflictWhenServiceThrowsRecordAuthorNotFound() throws Exception {
         final List<PatientRecord> records =
                 List.of(Generator.generatePatientRecord(user), Generator.generatePatientRecord(user));
         when(patientRecordServiceMock.importRecords(anyList())).thenThrow(RecordAuthorNotFoundException.class);
 
-        mockMvc.perform(post("/records/import").content(toJson(records)).contentType(MediaType.APPLICATION_JSON))
-               .andExpect(status().isConflict());
+        MockMultipartFile file = new MockMultipartFile("file", "records.json",
+                MediaType.MULTIPART_FORM_DATA_VALUE, toJson(records).getBytes());
+
+        mockMvc.perform(
+                multipart("/records/import/json")
+                        .file(file)
+                        .contentType(MediaType.MULTIPART_FORM_DATA_VALUE)
+        ).andExpect(status().isConflict());
     }
 
     @Test