[frontend/backend] Import a xls chronogram as a scenario (#1229)

Co-authored-by: Johanah LEKEU <[email protected]>
Co-authored-by: Romuald Lemesle <[email protected]>

openbas-api/pom.xml

@@ -22,6 +22,7 @@
         <springdoc.version>2.5.0</springdoc.version>
         <springdoc-plugin.version>1.4</springdoc-plugin.version>
         <cron-utils.version>9.2.1</cron-utils.version>
+        <apache-poi.version>5.3.0</apache-poi.version>
     </properties>
 
     <profiles>
@@ -167,6 +168,16 @@
             <artifactId>cron-utils</artifactId>
             <version>${cron-utils.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi</artifactId>
+            <version>${apache-poi.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.poi</groupId>
+            <artifactId>poi-ooxml</artifactId>
+            <version>${apache-poi.version}</version>
+        </dependency>
         <!-- TEST -->
         <dependency>
             <groupId>org.springframework.boot</groupId>

openbas-api/src/main/java/io/openbas/migration/V3_29__Add_tables_xls_mappers.java

@@ -0,0 +1,70 @@
+package io.openbas.migration;
+
+import org.flywaydb.core.api.migration.BaseJavaMigration;
+import org.flywaydb.core.api.migration.Context;
+import org.springframework.stereotype.Component;
+
+import java.sql.Connection;
+import java.sql.Statement;
+
+@Component
+public class V3_29__Add_tables_xls_mappers extends BaseJavaMigration {
+
+  @Override
+  public void migrate(Context context) throws Exception {
+    Connection connection = context.getConnection();
+    Statement select = connection.createStatement();
+    // Create table
+    select.execute("""
+          CREATE TABLE import_mappers (
+            mapper_id UUID NOT NULL CONSTRAINT import_mappers_pkey PRIMARY KEY,
+            mapper_name VARCHAR(255) NOT NULL,
+            mapper_inject_type_column VARCHAR(255) NOT NULL,
+            mapper_created_at TIMESTAMP DEFAULT now(),
+            mapper_updated_at TIMESTAMP DEFAULT now()
+            
+          );
+          CREATE INDEX idx_import_mappers ON import_mappers(mapper_id);
+     """);
+
+    select.execute("""
+          CREATE TABLE inject_importers (
+            importer_id UUID NOT NULL CONSTRAINT inject_importers_pkey PRIMARY KEY,
+            importer_mapper_id UUID NOT NULL
+              CONSTRAINT inject_importers_mapper_id_fkey REFERENCES import_mappers(mapper_id) ON DELETE SET NULL,
+            importer_import_type_value VARCHAR(255) NOT NULL,
+            importer_injector_contract_id VARCHAR(255) NOT NULL
+              CONSTRAINT inject_importers_injector_contract_id_fkey REFERENCES injectors_contracts(injector_contract_id) ON DELETE SET NULL,
+            importer_created_at TIMESTAMP DEFAULT now(),
+            importer_updated_at TIMESTAMP DEFAULT now()
+          );
+          CREATE INDEX idx_inject_importers ON inject_importers(importer_id);
+     """);
+
+
+    select.execute("""
+          CREATE TABLE rule_attributes (
+            attribute_id UUID NOT NULL CONSTRAINT rule_attributes_pkey PRIMARY KEY,
+            attribute_inject_importer_id UUID NOT NULL
+              CONSTRAINT rule_attributes_importer_id_fkey REFERENCES inject_importers(importer_id) ON DELETE SET NULL,
+            attribute_name varchar(255) not null,
+            attribute_columns varchar(255),
+            attribute_default_value varchar(255),
+            attribute_additional_config HSTORE,
+            attribute_created_at TIMESTAMP DEFAULT now(),
+            attribute_updated_at TIMESTAMP DEFAULT now()
+          );
+          CREATE INDEX idx_rule_attributes on rule_attributes(attribute_id);
+     """);
+
+
+    select.execute("""
+          ALTER TABLE injectors_contracts ADD COLUMN injector_contract_import_available BOOLEAN NOT NULL DEFAULT FALSE;
+     """);
+
+    select.execute("""
+          UPDATE injectors_contracts SET injector_contract_import_available = true WHERE injector_contract_labels -> 'en' LIKE ANY(ARRAY['%SMS%', '%Send%mail%']);
+     """);
+
+  }
+}

openbas-api/src/main/java/io/openbas/rest/exception/BadRequestException.java

@@ -0,0 +1,16 @@
+package io.openbas.rest.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseStatus(HttpStatus.BAD_REQUEST)
+public class BadRequestException extends RuntimeException{
+
+    public BadRequestException() {
+        super();
+    }
+
+    public BadRequestException(String errorMessage) {
+        super(errorMessage);
+    }
+}

openbas-api/src/main/java/io/openbas/rest/exception/FileTooBigException.java

@@ -0,0 +1,16 @@
+package io.openbas.rest.exception;
+
+import org.springframework.http.HttpStatus;
+import org.springframework.web.bind.annotation.ResponseStatus;
+
+@ResponseStatus(HttpStatus.BAD_REQUEST)
+public class FileTooBigException extends RuntimeException{
+
+    public FileTooBigException() {
+        super();
+    }
+
+    public FileTooBigException(String errorMessage) {
+        super(errorMessage);
+    }
+}

openbas-api/src/main/java/io/openbas/rest/helper/RestBehavior.java

@@ -8,24 +8,32 @@
 import io.openbas.database.model.Organization;
 import io.openbas.database.model.User;
 import io.openbas.database.repository.UserRepository;
+import io.openbas.rest.exception.ElementNotFoundException;
+import io.openbas.rest.exception.FileTooBigException;
 import io.openbas.rest.exception.InputValidationException;
+import jakarta.annotation.Resource;
+import lombok.extern.java.Log;
 import org.hibernate.exception.ConstraintViolationException;
+import org.springdoc.api.ErrorMessage;
 import org.springframework.dao.DataIntegrityViolationException;
 import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.AccessDeniedException;
 import org.springframework.validation.FieldError;
 import org.springframework.web.bind.MethodArgumentNotValidException;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.ResponseStatus;
+import org.springframework.web.bind.annotation.RestControllerAdvice;
+import org.springframework.web.reactive.function.UnsupportedMediaTypeException;
 
-import jakarta.annotation.Resource;
 import java.util.*;
 import java.util.stream.Collectors;
 
 import static io.openbas.config.OpenBASAnonymous.ANONYMOUS;
 import static io.openbas.config.SessionHelper.currentUser;
 
-
+@RestControllerAdvice
+@Log
 public class RestBehavior {
 
   @Resource
@@ -97,6 +105,27 @@ public ViolationErrorBag handleIntegrityException(DataIntegrityViolationExceptio
     return errorBag;
   }
 
+  @ExceptionHandler(ElementNotFoundException.class)
+  public ResponseEntity<ErrorMessage> handleElementNotFoundException(ElementNotFoundException ex) {
+    ErrorMessage message = new ErrorMessage("Element not found: " + ex.getMessage());
+    log.warning("ElementNotFoundException: " + ex.getMessage());
+    return new ResponseEntity<>(message, HttpStatus.NOT_FOUND);
+  }
+
+  @ExceptionHandler(UnsupportedMediaTypeException.class)
+  public ResponseEntity<ErrorMessage> handleUnsupportedMediaTypeException(UnsupportedMediaTypeException ex) {
+    ErrorMessage message = new ErrorMessage(ex.getMessage());
+    log.warning("UnsupportedMediaTypeException: " + ex.getMessage());
+    return new ResponseEntity<>(message, HttpStatus.UNSUPPORTED_MEDIA_TYPE);
+  }
+
+  @ExceptionHandler(FileTooBigException.class)
+  public ResponseEntity<ErrorMessage> handleFileTooBigException(FileTooBigException ex) {
+    ErrorMessage message = new ErrorMessage(ex.getMessage());
+    log.warning("FileTooBigException: " + ex.getMessage());
+    return new ResponseEntity<>(message, HttpStatus.BAD_REQUEST);
+  }
+
   // --- Open channel access
   public User impersonateUser(UserRepository userRepository, Optional<String> userId) {
     if (currentUser().getId().equals(ANONYMOUS)) {

openbas-api/src/main/java/io/openbas/rest/mapper/MapperApi.java

@@ -0,0 +1,131 @@
+package io.openbas.rest.mapper;
+
+import io.openbas.database.model.ImportMapper;
+import io.openbas.database.model.Scenario;
+import io.openbas.database.raw.RawPaginationImportMapper;
+import io.openbas.database.repository.ImportMapperRepository;
+import io.openbas.rest.exception.ElementNotFoundException;
+import io.openbas.rest.exception.FileTooBigException;
+import io.openbas.rest.helper.RestBehavior;
+import io.openbas.rest.mapper.form.ImportMapperAddInput;
+import io.openbas.rest.mapper.form.ImportMapperUpdateInput;
+import io.openbas.rest.scenario.form.InjectsImportTestInput;
+import io.openbas.rest.scenario.response.ImportPostSummary;
+import io.openbas.rest.scenario.response.ImportTestSummary;
+import io.openbas.service.InjectService;
+import io.openbas.service.MapperService;
+import io.openbas.utils.pagination.SearchPaginationInput;
+import io.swagger.v3.oas.annotations.Operation;
+import jakarta.transaction.Transactional;
+import jakarta.validation.Valid;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import lombok.RequiredArgsConstructor;
+import org.apache.commons.io.FilenameUtils;
+import org.springframework.data.domain.Page;
+import org.springframework.security.access.annotation.Secured;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.reactive.function.UnsupportedMediaTypeException;
+
+import java.time.Instant;
+import java.util.List;
+import java.util.UUID;
+
+import static io.openbas.database.model.User.ROLE_ADMIN;
+import static io.openbas.database.model.User.ROLE_USER;
+import static io.openbas.utils.pagination.PaginationUtils.buildPaginationJPA;
+
+@RestController
+@RequiredArgsConstructor
+public class MapperApi extends RestBehavior {
+
+    private final ImportMapperRepository importMapperRepository;
+
+    private final MapperService mapperService;
+
+    private final InjectService injectService;
+
+    // 25mb in byte
+    private static final int MAXIMUM_FILE_SIZE_ALLOWED = 25 * 1000 * 1000;
+    private static final List<String> ACCEPTED_FILE_TYPES = List.of("xls", "xlsx");
+
+    @Secured(ROLE_USER)
+    @PostMapping("/api/mappers/search")
+    public Page<RawPaginationImportMapper> getImportMapper(@RequestBody @Valid final SearchPaginationInput searchPaginationInput) {
+        return buildPaginationJPA(
+                this.importMapperRepository::findAll,
+                searchPaginationInput,
+                ImportMapper.class
+        ).map(RawPaginationImportMapper::new);
+    }
+
+    @Secured(ROLE_USER)
+    @GetMapping("/api/mappers/{mapperId}")
+    public ImportMapper getImportMapperById(@PathVariable String mapperId) {
+        return importMapperRepository.findById(UUID.fromString(mapperId)).orElseThrow(ElementNotFoundException::new);
+    }
+
+    @Secured(ROLE_ADMIN)
+    @PostMapping("/api/mappers")
+    public ImportMapper createImportMapper(@RequestBody @Valid final ImportMapperAddInput importMapperAddInput) {
+        return mapperService.createAndSaveImportMapper(importMapperAddInput);
+    }
+
+    @Secured(ROLE_ADMIN)
+    @PutMapping("/api/mappers/{mapperId}")
+    public ImportMapper updateImportMapper(@PathVariable String mapperId, @Valid @RequestBody ImportMapperUpdateInput importMapperUpdateInput) {
+        return mapperService.updateImportMapper(mapperId, importMapperUpdateInput);
+    }
+
+    @Secured(ROLE_ADMIN)
+    @DeleteMapping("/api/mappers/{mapperId}")
+    public void deleteImportMapper(@PathVariable String mapperId) {
+        importMapperRepository.deleteById(UUID.fromString(mapperId));
+    }
+
+    @PostMapping("/api/mappers/store")
+    @Transactional(rollbackOn = Exception.class)
+    @Operation(summary = "Import injects into an xls file")
+    @Secured(ROLE_USER)
+    public ImportPostSummary importXLSFile(@RequestPart("file") @NotNull MultipartFile file) {
+        validateUploadedFile(file);
+        return injectService.storeXlsFileForImport(file);
+    }
+
+    @PostMapping("/api/mappers/store/{importId}")
+    @Transactional(rollbackOn = Exception.class)
+    @Operation(summary = "Test the import of injects from an xls file")
+    @Secured(ROLE_USER)
+    public ImportTestSummary testImportXLSFile(@PathVariable @NotBlank final String importId,
+                                               @Valid @RequestBody final InjectsImportTestInput input) {
+        ImportMapper importMapper = mapperService.createImportMapper(input.getImportMapper());
+        importMapper.getInjectImporters().forEach(
+                injectImporter -> {
+                    injectImporter.setId(UUID.randomUUID().toString());
+                    injectImporter.getRuleAttributes().forEach(ruleAttribute -> ruleAttribute.setId(UUID.randomUUID().toString()));
+                }
+        );
+        Scenario scenario = new Scenario();
+        scenario.setRecurrenceStart(Instant.now());
+        return injectService.importInjectIntoScenarioFromXLS(scenario, importMapper, importId, input.getName(), input.getTimezoneOffset(), false);
+    }
+
+    private void validateUploadedFile(MultipartFile file) {
+        validateExtension(file);
+        validateFileSize(file);
+    }
+
+    private void validateExtension(MultipartFile file) {
+        String extension = FilenameUtils.getExtension(file.getOriginalFilename());
+        if (!ACCEPTED_FILE_TYPES.contains(extension)) {
+            throw new UnsupportedMediaTypeException("Only the following file types are accepted : " + String.join(", ", ACCEPTED_FILE_TYPES));
+        }
+    }
+
+    private void validateFileSize(MultipartFile file){
+        if (file.getSize() >= MAXIMUM_FILE_SIZE_ALLOWED) {
+            throw new FileTooBigException("File size cannot be greater than 25 Mb");
+        }
+    }
+}

openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperAddInput.java

@@ -0,0 +1,30 @@
+package io.openbas.rest.mapper.form;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import jakarta.validation.constraints.NotBlank;
+import jakarta.validation.constraints.NotNull;
+import jakarta.validation.constraints.Pattern;
+import lombok.Data;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static io.openbas.config.AppConfig.MANDATORY_MESSAGE;
+
+@Data
+public class ImportMapperAddInput {
+
+    @NotBlank(message = MANDATORY_MESSAGE)
+    @JsonProperty("mapper_name")
+    private String name;
+
+    @Pattern(regexp="^[A-Z]{1,2}$")
+    @JsonProperty("mapper_inject_type_column")
+    @NotBlank
+    private String injectTypeColumn;
+
+    @JsonProperty("mapper_inject_importers")
+    @NotNull
+    private List<InjectImporterAddInput> importers = new ArrayList<>();
+
+}