[BUG] 🐛 Export should return less than maxSizeOutput (#506)

docs/configuration.md

@@ -9,6 +9,7 @@
     - [Security configuration](#security-configuration)
     - [WebHooks configuration](#webhooks-configuration)
     - [Spring actuator configuration](#spring-actuator-configuration)
+    - [Export configuration](#export-configuration)
     - [Other info configuration](#other-info-configuration)
     - [Old endpoints configuration](#old-endpoints-configuration)
 
@@ -160,6 +161,15 @@ fr.insee.sugoi.security.monitor-user-password=monitor
 
 This user only has rights on `/actuator` endpoints.
 
+### Export configuration
+
+Configure the /export.csv endpoint with :
+
+| Properties                                   | Description | Default value |
+| ---------------------------------------------| :---------: | ------------: |
+| fr.insee.sugoi.export.maxSizeOutput          | The export function will stop when the number of entities reach this limit   | 10000 |
+| fr.insee.sugoi.export.pagesize               | The maximum size of pages to request on the store provider when exporting. With a pagesize of 10 the store will be requested for pages 10 by 10 before aggregating the results   | 2000  |
+
 ### Other info configuration
 
 You can add all other spring properties for example :

sugoi-api-ldap-store-provider/src/test/java/fr/insee/sugoi/ldap/LdapReaderStoreTest.java

@@ -31,6 +31,7 @@
 import java.util.List;
 import java.util.Map;
 import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -222,6 +223,18 @@ public void testSearchAllUsers() {
         users.stream().anyMatch(user -> user.getUsername().equals("testc")));
   }
 
+  @Test
+  @DisplayName(
+      "When the size of the pagerequest is less than the total result, the pageresult should have isHasMoreResult")
+  public void testAllUsersHasMoreResult() {
+    PageableResult pageableResult = new PageableResult();
+    pageableResult.setSize(1);
+    PageResult<User> result = ldapReaderStore.searchUsers(new User(), pageableResult, "AND");
+    assertThat("Should get one result", 1, is(result.getResults().size()));
+    assertThat("Page size should still be 1", 1, is(result.getResults().size()));
+    assertThat("Should be tagged with more results", result.isHasMoreResult());
+  }
+
   @Test
   public void testSearchUserWithMatchingMail() {
     PageableResult pageableResult = new PageableResult();

sugoi-api-rest-services/src/main/java/fr/insee/sugoi/services/controller/ExportController.java

@@ -143,10 +143,9 @@ public void getAllUsers(
     }
 
     int allResultsSize = 0;
-    try {
+    try (CSVPrinter csvPrinter = new CSVPrinter(response.getWriter(), CSVFormat.DEFAULT)) {
       response.setCharacterEncoding("UTF-8");
       response.setContentType("text/csv");
-      CSVPrinter csvPrinter = new CSVPrinter(response.getWriter(), CSVFormat.DEFAULT);
       // print headers on the first line
       csvPrinter.printRecord(getFieldsAsStringList(new User()));
 
@@ -165,10 +164,10 @@ public void getAllUsers(
           csvPrinter.flush();
 
           if (foundUsers.isHasMoreResult()) {
-            if (allResultsSize > maxSizeOutput) {
-              csvPrinter.close();
-              throw new RuntimeException("too much result");
+            if (allResultsSize + pageSize >= maxSizeOutput) {
+              break;
             }
+            allResultsSize += foundUsers.getResults().size();
             pageable = new PageableResult(pageSize, 0, foundUsers.getSearchToken());
           } else {
             break;

sugoi-api-rest-services/src/test/java/fr/insee/sugoi/services/controller/ExportControllerTest.java

@@ -0,0 +1,188 @@
+/*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package fr.insee.sugoi.services.controller;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThanOrEqualTo;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.lessThan;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import fr.insee.sugoi.commons.services.controller.technics.SugoiAdviceController;
+import fr.insee.sugoi.core.service.ConfigService;
+import fr.insee.sugoi.core.service.UserService;
+import fr.insee.sugoi.model.Realm;
+import fr.insee.sugoi.model.User;
+import fr.insee.sugoi.model.UserStorage;
+import fr.insee.sugoi.model.paging.PageResult;
+import fr.insee.sugoi.model.paging.PageableResult;
+import fr.insee.sugoi.model.paging.SearchType;
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.mock.web.MockHttpServletResponse;
+import org.springframework.security.test.context.support.WithMockUser;
+import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.RequestBuilder;
+import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
+import org.springframework.web.servlet.config.annotation.EnableWebMvc;
+
+@SpringBootTest(
+    classes = {ExportController.class, SugoiAdviceController.class},
+    properties = "spring.config.location=classpath:/controller/application.properties")
+@AutoConfigureMockMvc
+@EnableWebMvc
+public class ExportControllerTest {
+
+  @Autowired MockMvc mockMvc;
+
+  @MockBean private UserService userService;
+
+  @MockBean private ConfigService configService;
+
+  ObjectMapper objectMapper = new ObjectMapper();
+
+  @BeforeEach
+  public void setup() {
+    Realm realm = new Realm();
+    realm.setName("domaine1");
+    UserStorage userStorage = new UserStorage();
+    userStorage.setName("default");
+    realm.setUserStorages(List.of(userStorage));
+  }
+
+  @Test
+  @WithMockUser
+  @DisplayName("When there are only two users in userstorage all users should be returned")
+  public void getAllUsersWhenTotalIs2() throws Exception {
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("domaine1"),
+                Mockito.eq("default"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tutu", false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/domaine1/storages/default/export/users/export.csv");
+    MockHttpServletResponse response = mockMvc.perform(requestBuilder).andReturn().getResponse();
+
+    assertThat(
+        "HttpServletResponse type should be a csv",
+        "text/csv;charset=UTF-8",
+        is(response.getContentType()));
+    List<String> usernames = convertCsvToUsernames(response.getContentAsString());
+    assertThat("Should return 2 users", 2, is(usernames.size()));
+    assertThat(
+        "Should have a user tata",
+        usernames.stream().anyMatch(username -> username.equals("tutu")));
+  }
+
+  @Test
+  @WithMockUser
+  @DisplayName(
+      "When there are more users than the pagesize, "
+          + "but less than the maximum value then "
+          + "users should be requested several times and all users should be returned")
+  public void getTotalUsersWhenMoreThanPageSize() throws Exception {
+
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("domaine1"),
+                Mockito.eq("default"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tata", true))
+        .thenReturn(createPageResult("tutu", null, false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/domaine1/storages/default/export/users/export.csv");
+    List<String> usernames =
+        convertCsvToUsernames(
+            mockMvc.perform(requestBuilder).andReturn().getResponse().getContentAsString());
+
+    assertThat("Should return 3 users even if page size is 2", 3, is(usernames.size()));
+  }
+
+  @Test
+  @WithMockUser
+  @DisplayName(
+      "When there are more users than the maximum a user can fetch, max exception should be returned")
+  public void exceptionIsReturnedWhenMoreThanMax() throws Exception {
+
+    Mockito.when(
+            userService.findByProperties(
+                Mockito.eq("domaine1"),
+                Mockito.eq("default"),
+                Mockito.isA(User.class),
+                Mockito.isA(PageableResult.class),
+                Mockito.isA(SearchType.class)))
+        .thenReturn(createPageResult("toto", "tata", true))
+        .thenReturn(createPageResult("tutu", "toto2", true))
+        .thenReturn(createPageResult("toto4", "toto4", true))
+        .thenReturn(createPageResult("toto5", "toto6", true))
+        .thenReturn(createPageResult("toto7", "toto8", true))
+        .thenReturn(createPageResult("toto9", "toto10", true))
+        .thenReturn(createPageResult("toto11", "toto12", false));
+
+    RequestBuilder requestBuilder =
+        MockMvcRequestBuilders.get("/realms/domaine1/storages/default/export/users/export.csv");
+    List<String> usernames =
+        convertCsvToUsernames(
+            mockMvc.perform(requestBuilder).andReturn().getResponse().getContentAsString());
+
+    assertThat("Should return a value under the max", 10, greaterThanOrEqualTo(usernames.size()));
+    assertThat(
+        "Should return a value above the max plus the pagesize",
+        10 - 2,
+        lessThan(usernames.size()));
+  }
+
+  private List<String> convertCsvToUsernames(String usersCsv) throws UnsupportedEncodingException {
+    List<String> usernames =
+        Arrays.stream(usersCsv.split("\n"))
+            .skip(1)
+            .map(line -> line.split(",")[3])
+            .collect(Collectors.toList());
+    return usernames;
+  }
+
+  private PageResult<User> createPageResult(
+      String firstName, String secondName, boolean hasMoreResult) {
+    PageResult<User> usersResult = new PageResult<>();
+    usersResult.setPageSize(2);
+    usersResult.setHasMoreResult(hasMoreResult);
+    List<User> users = new ArrayList<>();
+    if (firstName != null) {
+      users.add(new User(firstName));
+    }
+    if (secondName != null) {
+      users.add(new User(secondName));
+    }
+    usersResult.setResults(users);
+    return usersResult;
+  }
+}

sugoi-api-rest-services/src/test/resources/controller/application.properties

@@ -1,2 +1,3 @@
-
-logging.level.fr.insee.sugoi=debug
+logging.level.fr.insee.sugoi=debug
+fr.insee.sugoi.export.maxSizeOutput=10
+fr.insee.sugoi.export.pagesize=2