[FEA] ✨ Users can be searched ignoring accents

Signed-off-by: Cécile Chemin <[email protected]>

docs/configuration.md

@@ -64,6 +64,7 @@ For each realm we have the possibility to configure a default reader and a defau
 | fr.insee.sugoi.ldap.default.application-mapping                      |                                                  List of mappings between sugoi application attributes and ldap attributes divided by semicolon, see [Realm configuration](realm-configuration.md)                                                   |                                                                                                   name:ou,String,rw |
 | fr.insee.sugoi.id-create-length                                      |                                                                                                          Size of the ids randomly generated                                                                                                          |                                                                                                                   7 |
 | fr.insee.sugoi.reader-store-asynchronous                             | Is the reader store asynchronous, ie a difference can exist between what we read in readerstore and the realty. Can occur if the current service is connected by a broker to the real service. If true MAIL and ID unicity control are NOT performed |                                                                                                               false |
+| fr.insee.sugoi.fuzzy-search-allowed                                  |                                                                               If fuzzy search allowed, the user can ask to make an extensive request ignoring accents.                                                                               |                                                                                                               false |
 | fr.insee.sugoi.users.maxoutputsize                                   |                                                                                                  The default maximum number of user outputs allowed                                                                                                  |                                                                                                                1000 |                  100 |
 | fr.insee.sugoi.groups.maxoutputsize                                  |                                                                                                 The default maximum number of groups outputs allowed                                                                                                 |                                                                                                                1000 |                  100 |
 | fr.insee.sugoi.organizations.maxoutputsize                           |                                                                                             The default maximum number of organizations outputs allowed                                                                                              |                                                                                                                1000 |                  100 |

sugoi-api-core/src/main/java/fr/insee/sugoi/core/service/UserService.java

@@ -85,14 +85,17 @@ public interface UserService {
    * @param userProperties
    * @param pageable
    * @param typeRecherche
+   * @param fuzzySearchEnabled if true, the user common name should be searched without taking into
+   *     account accents, spaces and other specials characters
    * @return a list of users
    */
   PageResult<User> findByProperties(
       String realm,
       String storageName,
       User userProperties,
       PageableResult pageable,
-      SearchType typeRecherche);
+      SearchType typeRecherche,
+      boolean fuzzySearchEnabled);
 
   /**
    * Allow to add only the app-managed attribute of an user, this attribute must follow the

sugoi-api-core/src/main/java/fr/insee/sugoi/core/service/impl/UserServiceImpl.java

@@ -22,6 +22,7 @@
 import fr.insee.sugoi.core.realm.RealmProvider;
 import fr.insee.sugoi.core.seealso.SeeAlsoService;
 import fr.insee.sugoi.core.service.UserService;
+import fr.insee.sugoi.core.store.ReaderStore;
 import fr.insee.sugoi.core.store.StoreProvider;
 import fr.insee.sugoi.model.Realm;
 import fr.insee.sugoi.model.User;
@@ -39,6 +40,7 @@
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.stream.Collectors;
 import org.passay.CharacterRule;
 import org.passay.PasswordGenerator;
 import org.slf4j.Logger;
@@ -65,6 +67,8 @@ public class UserServiceImpl implements UserService {
    */
   private boolean readerStoreAsynchronous = false;
 
+  private boolean fuzzySearchAllowed = false;
+
   @Autowired private StoreProvider storeProvider;
 
   @Autowired private RealmProvider realmProvider;
@@ -228,7 +232,8 @@ public PageResult<User> findByProperties(
       String storage,
       User userProperties,
       PageableResult pageable,
-      SearchType typeRecherche) {
+      SearchType typeRecherche,
+      boolean fuzzySearchEnabled) {
 
     PageResult<User> result = new PageResult<>();
     Realm r = realmProvider.load(realm).orElseThrow(() -> new RealmNotFoundException(realm));
@@ -241,43 +246,34 @@ public PageResult<User> findByProperties(
                 .get(0)));
     result.setPageSize(pageable.getSize());
 
-    if (storage != null) {
-      result =
-          storeProvider
-              .getReaderStore(realm, storage)
-              .searchUsers(userProperties, pageable, typeRecherche.name());
-      result
+    List<String> userStoragesToBrowse =
+        storage != null
+            ? List.of(storage)
+            : r.getUserStorages().stream().map(UserStorage::getName).collect(Collectors.toList());
+    for (String usName : userStoragesToBrowse) {
+      ReaderStore readerStore = storeProvider.getReaderStore(realm, usName);
+      PageResult<User> temResult =
+          fuzzySearchEnabled && fuzzySearchAllowed
+              ? readerStore.fuzzySearchUsers(userProperties, pageable, typeRecherche.name())
+              : readerStore.searchUsers(userProperties, pageable, typeRecherche.name());
+      temResult
           .getResults()
           .forEach(
               user -> {
                 user.addMetadatas(EventKeysConfig.REALM, realm);
-                user.addMetadatas(EventKeysConfig.USERSTORAGE, storage);
+                user.addMetadatas(EventKeysConfig.USERSTORAGE, usName);
               });
-    } else {
-      for (UserStorage us : r.getUserStorages()) {
-        PageResult<User> temResult =
-            storeProvider
-                .getReaderStore(realm, us.getName())
-                .searchUsers(userProperties, pageable, typeRecherche.name());
-        temResult
-            .getResults()
-            .forEach(
-                user -> {
-                  user.addMetadatas(EventKeysConfig.REALM, realm);
-                  user.addMetadatas(EventKeysConfig.USERSTORAGE, us.getName());
-                });
-        result.getResults().addAll(temResult.getResults());
-        result.setTotalElements(
-            temResult.getTotalElements() == -1
-                ? temResult.getTotalElements()
-                : result.getTotalElements() + temResult.getTotalElements());
-        result.setSearchToken(temResult.getSearchToken());
-        result.setHasMoreResult(temResult.isHasMoreResult());
-        if (result.getResults().size() >= result.getPageSize()) {
-          return result;
-        }
-        pageable.setSize(pageable.getSize() - result.getTotalElements());
+      result.getResults().addAll(temResult.getResults());
+      result.setTotalElements(
+          temResult.getTotalElements() == -1
+              ? temResult.getTotalElements()
+              : result.getTotalElements() + temResult.getTotalElements());
+      result.setSearchToken(temResult.getSearchToken());
+      result.setHasMoreResult(temResult.isHasMoreResult());
+      if (result.getResults().size() >= result.getPageSize()) {
+        return result;
       }
+      pageable.setSize(pageable.getSize() - result.getTotalElements());
     }
 
     return result;

sugoi-api-core/src/main/java/fr/insee/sugoi/core/store/ReaderStore.java

@@ -59,6 +59,19 @@ public interface ReaderStore {
   public PageResult<User> searchUsers(
       User userFilter, PageableResult pageable, String searchOperator);
 
+  /**
+   * Search users matching userFilter filled attributes. More results are returned than normal
+   * search since accents and other special characters are ignored.
+   *
+   * @param userFilter an incomplete user with attributes set to be matched with
+   * @param pageable properties for pageable request
+   * @param searchOperator 'OR' or 'AND' to determine if multiple attributes should match or only
+   *     one
+   * @return a PageResult containing a list of matching users.
+   */
+  public PageResult<User> fuzzySearchUsers(
+      User userFilter, PageableResult pageable, String searchOperator);
+
   /**
    * Retrieve the organization with the given id in the store.
    *

sugoi-api-core/src/test/java/fr/insee/sugoi/core/service/UserServiceTest.java

@@ -190,7 +190,12 @@ public void findUsersShouldFailWhenRealmNotFound() {
         RealmNotFoundException.class,
         () ->
             userService.findByProperties(
-                "idonotexist", "us2", new User("Toto"), new PageableResult(), SearchType.AND));
+                "idonotexist",
+                "us2",
+                new User("Toto"),
+                new PageableResult(),
+                SearchType.AND,
+                false));
   }
 
   @Test
@@ -309,7 +314,7 @@ public void getUsersWithPageableSizeShouldReturnEnoughUsers() {
     PageableResult pageable = new PageableResult(30000, 0, null);
     List<User> results =
         userService
-            .findByProperties(realm.getName(), null, new User(), pageable, SearchType.AND)
+            .findByProperties(realm.getName(), null, new User(), pageable, SearchType.AND, false)
             .getResults();
     assertThat("30000 users are retrieved", results.size(), is(30000));
     assertThat(

sugoi-api-file-store-provider/src/main/java/fr/insee/sugoi/store/file/FileReaderStore.java

@@ -94,6 +94,12 @@ public PageResult<User> searchUsers(
     return pageResult;
   }
 
+  @Override
+  public PageResult<User> fuzzySearchUsers(
+      User userFilter, PageableResult pageable, String searchOperator) {
+    throw new NotImplementedException();
+  }
+
   @Override
   public Optional<Organization> getOrganization(String id) {
     if (StringUtils.isNotBlank(config.get(GlobalKeysConfig.ORGANIZATION_SOURCE))) {

sugoi-api-ldap-store-provider/src/main/java/fr/insee/sugoi/store/ldap/LdapReaderStore.java

@@ -40,9 +40,11 @@
 import fr.insee.sugoi.model.paging.PageableResult;
 import fr.insee.sugoi.model.paging.SearchType;
 import fr.insee.sugoi.model.technics.StoreMapping;
+import java.text.Normalizer;
 import java.util.*;
 import java.util.stream.Collectors;
 import org.apache.commons.lang.StringUtils;
+import org.springframework.util.Assert;
 
 public class LdapReaderStore extends LdapStore implements ReaderStore {
 
@@ -130,6 +132,46 @@ public PageResult<User> searchUsers(
     }
   }
 
+  @Override
+  public PageResult<User> fuzzySearchUsers(
+      User userFilter, PageableResult pageable, String typeRecherche) {
+    String initialCommonName = (String) userFilter.getAttributes().get("common_name");
+    if (initialCommonName == null) {
+      return searchUsers(userFilter, pageable, typeRecherche);
+    } else {
+      try {
+        userFilter
+            .getAttributes()
+            .put(
+                "common_name",
+                initialCommonName
+                    .replaceAll(
+                        "[ÀÁÂÃÄAÅÇCÈÉÊËEÌÍIÎÏÐÒÓÔOÕÖÙUÚÛÜÝYŸàáâãäåçèéêëìíîïðòóôõöùúûüýÿaeiouc \\-']",
+                        "*")
+                    .replaceAll("\\*+", "*"));
+        PageResult<User> results =
+            searchOnLdap(
+                config.get(GlobalKeysConfig.USER_SOURCE),
+                SearchScope.SUBORDINATE_SUBTREE,
+                getFilterFromObject(userFilter, userLdapMapper, typeRecherche, false),
+                pageable,
+                userLdapMapper);
+        String normalizedCommonName = removeSpecialChars(initialCommonName);
+        List<User> filteredUsers =
+            results.getResults().stream()
+                .filter(
+                    u ->
+                        removeSpecialChars((String) u.getAttributes().get("common_name"))
+                            .equalsIgnoreCase(normalizedCommonName))
+                .collect(Collectors.toList());
+        results.setResults(filteredUsers);
+        return results;
+      } catch (LDAPException e) {
+        throw new StoreException("Fail to execute user search", e);
+      }
+    }
+  }
+
   @Override
   public PageResult<User> getUsersInGroup(String appName, String groupName) {
     PageResult<User> page = new PageResult<>();
@@ -249,6 +291,11 @@ public PageResult<Application> searchApplications(
     }
   }
 
+  private <M extends SugoiObject> Filter getFilterFromObject(
+      M object, LdapMapper<M> mapper, String searchType) {
+    return getFilterFromObject(object, mapper, searchType, true);
+  }
+
   /**
    * Create a filter from an object using a mapper class. Each set field of the object is
    * transformed to a filter.
@@ -259,40 +306,68 @@ public PageResult<Application> searchApplications(
    * @return a filter corresponding to the properties of object
    */
   private <M extends SugoiObject> Filter getFilterFromObject(
-      M object, LdapMapper<M> mapper, String searchType) {
-    if (searchType.equalsIgnoreCase("AND")) {
+      M object, LdapMapper<M> mapper, String searchType, boolean encodeCommonNameWildcard) {
+    Assert.isTrue(
+        searchType.equals("AND") || searchType.equals("OR"), "Search type should be AND or OR.");
+    List<Attribute> attributes = mapper.createAttributesForFilter(object);
+    List<Filter> attributeListFilter = getAttributesFilters(attributes, encodeCommonNameWildcard);
+    List<Filter> objectClassListFilter = getObjectClassFilters(attributes);
+    if (!objectClassListFilter.isEmpty() && attributeListFilter.isEmpty()) {
+      return LdapFilter.and(objectClassListFilter);
+    } else if (objectClassListFilter.isEmpty() && !attributeListFilter.isEmpty()) {
+      return searchType.equals("AND")
+          ? LdapFilter.or(attributeListFilter)
+          : LdapFilter.and(attributeListFilter);
+    } else {
       return LdapFilter.and(
-          mapper.createAttributesForFilter(object).stream()
-              .filter(attribute -> !attribute.getValue().equals(""))
-              .map(attribute -> LdapFilter.createFilter(attribute.getName(), attribute.getValues()))
-              .collect(Collectors.toList()));
-    } else if (searchType.equalsIgnoreCase("OR")) {
-      List<Filter> objectClassListFilter =
-          mapper.createAttributesForFilter(object).stream()
-              .filter(attribute -> attribute.getName().equals("objectClass"))
-              .map(attribute -> LdapFilter.createFilter(attribute.getName(), attribute.getValues()))
-              .collect(Collectors.toList());
+          Arrays.asList(
+              LdapFilter.and(objectClassListFilter),
+              searchType.equals("AND")
+                  ? LdapFilter.or(attributeListFilter)
+                  : LdapFilter.and(attributeListFilter)));
+    }
+  }
 
-      List<Filter> attributeListFilter =
-          mapper.createAttributesForFilter(object).stream()
-              .filter(
-                  attribute ->
-                      !attribute.getName().equals("objectClass")
-                          && !attribute.getValue().equals(""))
-              .map(attribute -> LdapFilter.createFilter(attribute.getName(), attribute.getValues()))
-              .collect(Collectors.toList());
+  private List<Filter> getObjectClassFilters(List<Attribute> attributes) {
+    return attributes.stream()
+        .filter(attribute -> attribute.getName().equals("objectClass"))
+        .map(attribute -> LdapFilter.createFilter(attribute.getName(), attribute.getValues()))
+        .collect(Collectors.toList());
+  }
 
-      if (!objectClassListFilter.isEmpty() && attributeListFilter.isEmpty()) {
-        return LdapFilter.and(objectClassListFilter);
-      } else if (objectClassListFilter.isEmpty() && !attributeListFilter.isEmpty()) {
-        return LdapFilter.or(attributeListFilter);
+  private List<Filter> getAttributesFilters(
+      List<Attribute> attributes, boolean encodeCommonNameWildcard) {
+    List<Filter> filters =
+        attributes.stream()
+            .filter(
+                attribute ->
+                    !attribute.getName().equals("objectClass")
+                        && !attribute.getValue().isEmpty()
+                        && !attribute.getName().equals("cn"))
+            .map(attribute -> LdapFilter.createFilter(attribute.getName(), attribute.getValues()))
+            .collect(Collectors.toList());
+    Optional<Attribute> commonNameAttribute =
+        attributes.stream().filter(attribute -> attribute.getName().equals("cn")).findFirst();
+    if (commonNameAttribute.isPresent()) {
+      if (encodeCommonNameWildcard) {
+        filters.add(
+            LdapFilter.createFilter(
+                commonNameAttribute.get().getName(), commonNameAttribute.get().getValues()));
       } else {
-        return LdapFilter.and(
-            Arrays.asList(
-                LdapFilter.and(objectClassListFilter), LdapFilter.or(attributeListFilter)));
+        try {
+          filters.add(
+              Filter.create(
+                  "cn="
+                      + Filter.encodeValue(commonNameAttribute.get().getValue())
+                          .replace("\\2a", "*")));
+        } catch (LDAPException e) {
+          filters.add(
+              LdapFilter.createFilter(
+                  commonNameAttribute.get().getName(), commonNameAttribute.get().getValues()));
+        }
       }
     }
-    throw new RuntimeException("Invalid searchType must be AND or OR");
+    return filters;
   }
 
   private SearchResultEntry getEntryByDn(String dn) {
@@ -420,4 +495,10 @@ private Optional<Organization> getOrganization(String id, boolean isSubOrganizat
     }
     return Optional.ofNullable(org);
   }
+
+  public String removeSpecialChars(String string) {
+    return Normalizer.normalize(string, Normalizer.Form.NFD)
+        .replaceAll("[-'\\s]+", "")
+        .replaceAll("\\p{InCombiningDiacriticalMarks}+", "");
+  }
 }