diff --git a/openbas-api/pom.xml b/openbas-api/pom.xml index a1cf14cdfd..2bba280e4e 100644 --- a/openbas-api/pom.xml +++ b/openbas-api/pom.xml @@ -22,6 +22,7 @@ 2.5.0 1.4 9.2.1 + 5.3.0 @@ -167,6 +168,16 @@ cron-utils ${cron-utils.version} + + org.apache.poi + poi + ${apache-poi.version} + + + org.apache.poi + poi-ooxml + ${apache-poi.version} + org.springframework.boot diff --git a/openbas-api/src/main/java/io/openbas/config/GlobalExceptionHandler.java b/openbas-api/src/main/java/io/openbas/config/GlobalExceptionHandler.java deleted file mode 100644 index 3539e496e0..0000000000 --- a/openbas-api/src/main/java/io/openbas/config/GlobalExceptionHandler.java +++ /dev/null @@ -1,22 +0,0 @@ -package io.openbas.config; - -import io.openbas.rest.exception.ElementNotFoundException; -import lombok.extern.java.Log; -import org.springdoc.api.ErrorMessage; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.ExceptionHandler; -import org.springframework.web.bind.annotation.RestControllerAdvice; - -@RestControllerAdvice -@Log -public class GlobalExceptionHandler { - - @ExceptionHandler(ElementNotFoundException.class) - public ResponseEntity handleElementNotFoundException(ElementNotFoundException ex) { - ErrorMessage message = new ErrorMessage("Element not found: " + ex.getMessage()); - log.warning("ElementNotFoundException: " + ex.getMessage()); - return new ResponseEntity<>(message, HttpStatus.NOT_FOUND); - } - -} diff --git a/openbas-api/src/main/java/io/openbas/migration/V3_29__Add_tables_xls_mappers.java b/openbas-api/src/main/java/io/openbas/migration/V3_29__Add_tables_xls_mappers.java new file mode 100644 index 0000000000..3f1878505a --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/migration/V3_29__Add_tables_xls_mappers.java @@ -0,0 +1,70 @@ +package io.openbas.migration; + +import org.flywaydb.core.api.migration.BaseJavaMigration; +import org.flywaydb.core.api.migration.Context; +import org.springframework.stereotype.Component; + +import java.sql.Connection; +import java.sql.Statement; + +@Component +public class V3_29__Add_tables_xls_mappers extends BaseJavaMigration { + + @Override + public void migrate(Context context) throws Exception { + Connection connection = context.getConnection(); + Statement select = connection.createStatement(); + // Create table + select.execute(""" + CREATE TABLE import_mappers ( + mapper_id UUID NOT NULL CONSTRAINT import_mappers_pkey PRIMARY KEY, + mapper_name VARCHAR(255) NOT NULL, + mapper_inject_type_column VARCHAR(255) NOT NULL, + mapper_created_at TIMESTAMP DEFAULT now(), + mapper_updated_at TIMESTAMP DEFAULT now() + + ); + CREATE INDEX idx_import_mappers ON import_mappers(mapper_id); + """); + + select.execute(""" + CREATE TABLE inject_importers ( + importer_id UUID NOT NULL CONSTRAINT inject_importers_pkey PRIMARY KEY, + importer_mapper_id UUID NOT NULL + CONSTRAINT inject_importers_mapper_id_fkey REFERENCES import_mappers(mapper_id) ON DELETE SET NULL, + importer_import_type_value VARCHAR(255) NOT NULL, + importer_injector_contract_id VARCHAR(255) NOT NULL + CONSTRAINT inject_importers_injector_contract_id_fkey REFERENCES injectors_contracts(injector_contract_id) ON DELETE SET NULL, + importer_created_at TIMESTAMP DEFAULT now(), + importer_updated_at TIMESTAMP DEFAULT now() + ); + CREATE INDEX idx_inject_importers ON inject_importers(importer_id); + """); + + + select.execute(""" + CREATE TABLE rule_attributes ( + attribute_id UUID NOT NULL CONSTRAINT rule_attributes_pkey PRIMARY KEY, + attribute_inject_importer_id UUID NOT NULL + CONSTRAINT rule_attributes_importer_id_fkey REFERENCES inject_importers(importer_id) ON DELETE SET NULL, + attribute_name varchar(255) not null, + attribute_columns varchar(255), + attribute_default_value varchar(255), + attribute_additional_config HSTORE, + attribute_created_at TIMESTAMP DEFAULT now(), + attribute_updated_at TIMESTAMP DEFAULT now() + ); + CREATE INDEX idx_rule_attributes on rule_attributes(attribute_id); + """); + + + select.execute(""" + ALTER TABLE injectors_contracts ADD COLUMN injector_contract_import_available BOOLEAN NOT NULL DEFAULT FALSE; + """); + + select.execute(""" + UPDATE injectors_contracts SET injector_contract_import_available = true WHERE injector_contract_labels -> 'en' LIKE ANY(ARRAY['%SMS%', '%Send%mail%']); + """); + + } +} \ No newline at end of file diff --git a/openbas-api/src/main/java/io/openbas/rest/exception/BadRequestException.java b/openbas-api/src/main/java/io/openbas/rest/exception/BadRequestException.java new file mode 100644 index 0000000000..3f3a0c9cd9 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/exception/BadRequestException.java @@ -0,0 +1,16 @@ +package io.openbas.rest.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.BAD_REQUEST) +public class BadRequestException extends RuntimeException{ + + public BadRequestException() { + super(); + } + + public BadRequestException(String errorMessage) { + super(errorMessage); + } +} diff --git a/openbas-api/src/main/java/io/openbas/rest/exception/FileTooBigException.java b/openbas-api/src/main/java/io/openbas/rest/exception/FileTooBigException.java new file mode 100644 index 0000000000..dc4867428d --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/exception/FileTooBigException.java @@ -0,0 +1,16 @@ +package io.openbas.rest.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.BAD_REQUEST) +public class FileTooBigException extends RuntimeException{ + + public FileTooBigException() { + super(); + } + + public FileTooBigException(String errorMessage) { + super(errorMessage); + } +} diff --git a/openbas-api/src/main/java/io/openbas/rest/helper/RestBehavior.java b/openbas-api/src/main/java/io/openbas/rest/helper/RestBehavior.java index 676aacd695..1a2fa3bf2e 100644 --- a/openbas-api/src/main/java/io/openbas/rest/helper/RestBehavior.java +++ b/openbas-api/src/main/java/io/openbas/rest/helper/RestBehavior.java @@ -8,24 +8,32 @@ import io.openbas.database.model.Organization; import io.openbas.database.model.User; import io.openbas.database.repository.UserRepository; +import io.openbas.rest.exception.ElementNotFoundException; +import io.openbas.rest.exception.FileTooBigException; import io.openbas.rest.exception.InputValidationException; +import jakarta.annotation.Resource; +import lombok.extern.java.Log; import org.hibernate.exception.ConstraintViolationException; +import org.springdoc.api.ErrorMessage; import org.springframework.dao.DataIntegrityViolationException; import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; import org.springframework.validation.FieldError; import org.springframework.web.bind.MethodArgumentNotValidException; import org.springframework.web.bind.annotation.ExceptionHandler; import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.reactive.function.UnsupportedMediaTypeException; -import jakarta.annotation.Resource; import java.util.*; import java.util.stream.Collectors; import static io.openbas.config.OpenBASAnonymous.ANONYMOUS; import static io.openbas.config.SessionHelper.currentUser; - +@RestControllerAdvice +@Log public class RestBehavior { @Resource @@ -97,6 +105,27 @@ public ViolationErrorBag handleIntegrityException(DataIntegrityViolationExceptio return errorBag; } + @ExceptionHandler(ElementNotFoundException.class) + public ResponseEntity handleElementNotFoundException(ElementNotFoundException ex) { + ErrorMessage message = new ErrorMessage("Element not found: " + ex.getMessage()); + log.warning("ElementNotFoundException: " + ex.getMessage()); + return new ResponseEntity<>(message, HttpStatus.NOT_FOUND); + } + + @ExceptionHandler(UnsupportedMediaTypeException.class) + public ResponseEntity handleUnsupportedMediaTypeException(UnsupportedMediaTypeException ex) { + ErrorMessage message = new ErrorMessage(ex.getMessage()); + log.warning("UnsupportedMediaTypeException: " + ex.getMessage()); + return new ResponseEntity<>(message, HttpStatus.UNSUPPORTED_MEDIA_TYPE); + } + + @ExceptionHandler(FileTooBigException.class) + public ResponseEntity handleFileTooBigException(FileTooBigException ex) { + ErrorMessage message = new ErrorMessage(ex.getMessage()); + log.warning("FileTooBigException: " + ex.getMessage()); + return new ResponseEntity<>(message, HttpStatus.BAD_REQUEST); + } + // --- Open channel access public User impersonateUser(UserRepository userRepository, Optional userId) { if (currentUser().getId().equals(ANONYMOUS)) { diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/MapperApi.java b/openbas-api/src/main/java/io/openbas/rest/mapper/MapperApi.java new file mode 100644 index 0000000000..14894a17e7 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/MapperApi.java @@ -0,0 +1,131 @@ +package io.openbas.rest.mapper; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.model.Scenario; +import io.openbas.database.raw.RawPaginationImportMapper; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.rest.exception.ElementNotFoundException; +import io.openbas.rest.exception.FileTooBigException; +import io.openbas.rest.helper.RestBehavior; +import io.openbas.rest.mapper.form.ImportMapperAddInput; +import io.openbas.rest.mapper.form.ImportMapperUpdateInput; +import io.openbas.rest.scenario.form.InjectsImportTestInput; +import io.openbas.rest.scenario.response.ImportPostSummary; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.service.InjectService; +import io.openbas.service.MapperService; +import io.openbas.utils.pagination.SearchPaginationInput; +import io.swagger.v3.oas.annotations.Operation; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.apache.commons.io.FilenameUtils; +import org.springframework.data.domain.Page; +import org.springframework.security.access.annotation.Secured; +import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.reactive.function.UnsupportedMediaTypeException; + +import java.time.Instant; +import java.util.List; +import java.util.UUID; + +import static io.openbas.database.model.User.ROLE_ADMIN; +import static io.openbas.database.model.User.ROLE_USER; +import static io.openbas.utils.pagination.PaginationUtils.buildPaginationJPA; + +@RestController +@RequiredArgsConstructor +public class MapperApi extends RestBehavior { + + private final ImportMapperRepository importMapperRepository; + + private final MapperService mapperService; + + private final InjectService injectService; + + // 25mb in byte + private static final int MAXIMUM_FILE_SIZE_ALLOWED = 25 * 1000 * 1000; + private static final List ACCEPTED_FILE_TYPES = List.of("xls", "xlsx"); + + @Secured(ROLE_USER) + @PostMapping("/api/mappers/search") + public Page getImportMapper(@RequestBody @Valid final SearchPaginationInput searchPaginationInput) { + return buildPaginationJPA( + this.importMapperRepository::findAll, + searchPaginationInput, + ImportMapper.class + ).map(RawPaginationImportMapper::new); + } + + @Secured(ROLE_USER) + @GetMapping("/api/mappers/{mapperId}") + public ImportMapper getImportMapperById(@PathVariable String mapperId) { + return importMapperRepository.findById(UUID.fromString(mapperId)).orElseThrow(ElementNotFoundException::new); + } + + @Secured(ROLE_ADMIN) + @PostMapping("/api/mappers") + public ImportMapper createImportMapper(@RequestBody @Valid final ImportMapperAddInput importMapperAddInput) { + return mapperService.createAndSaveImportMapper(importMapperAddInput); + } + + @Secured(ROLE_ADMIN) + @PutMapping("/api/mappers/{mapperId}") + public ImportMapper updateImportMapper(@PathVariable String mapperId, @Valid @RequestBody ImportMapperUpdateInput importMapperUpdateInput) { + return mapperService.updateImportMapper(mapperId, importMapperUpdateInput); + } + + @Secured(ROLE_ADMIN) + @DeleteMapping("/api/mappers/{mapperId}") + public void deleteImportMapper(@PathVariable String mapperId) { + importMapperRepository.deleteById(UUID.fromString(mapperId)); + } + + @PostMapping("/api/mappers/store") + @Transactional(rollbackOn = Exception.class) + @Operation(summary = "Import injects into an xls file") + @Secured(ROLE_USER) + public ImportPostSummary importXLSFile(@RequestPart("file") @NotNull MultipartFile file) { + validateUploadedFile(file); + return injectService.storeXlsFileForImport(file); + } + + @PostMapping("/api/mappers/store/{importId}") + @Transactional(rollbackOn = Exception.class) + @Operation(summary = "Test the import of injects from an xls file") + @Secured(ROLE_USER) + public ImportTestSummary testImportXLSFile(@PathVariable @NotBlank final String importId, + @Valid @RequestBody final InjectsImportTestInput input) { + ImportMapper importMapper = mapperService.createImportMapper(input.getImportMapper()); + importMapper.getInjectImporters().forEach( + injectImporter -> { + injectImporter.setId(UUID.randomUUID().toString()); + injectImporter.getRuleAttributes().forEach(ruleAttribute -> ruleAttribute.setId(UUID.randomUUID().toString())); + } + ); + Scenario scenario = new Scenario(); + scenario.setRecurrenceStart(Instant.now()); + return injectService.importInjectIntoScenarioFromXLS(scenario, importMapper, importId, input.getName(), input.getTimezoneOffset(), false); + } + + private void validateUploadedFile(MultipartFile file) { + validateExtension(file); + validateFileSize(file); + } + + private void validateExtension(MultipartFile file) { + String extension = FilenameUtils.getExtension(file.getOriginalFilename()); + if (!ACCEPTED_FILE_TYPES.contains(extension)) { + throw new UnsupportedMediaTypeException("Only the following file types are accepted : " + String.join(", ", ACCEPTED_FILE_TYPES)); + } + } + + private void validateFileSize(MultipartFile file){ + if (file.getSize() >= MAXIMUM_FILE_SIZE_ALLOWED) { + throw new FileTooBigException("File size cannot be greater than 25 Mb"); + } + } +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperAddInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperAddInput.java new file mode 100644 index 0000000000..72c9b9e560 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperAddInput.java @@ -0,0 +1,30 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class ImportMapperAddInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("mapper_name") + private String name; + + @Pattern(regexp="^[A-Z]{1,2}$") + @JsonProperty("mapper_inject_type_column") + @NotBlank + private String injectTypeColumn; + + @JsonProperty("mapper_inject_importers") + @NotNull + private List importers = new ArrayList<>(); + +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperUpdateInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperUpdateInput.java new file mode 100644 index 0000000000..1a7dfe4d37 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/ImportMapperUpdateInput.java @@ -0,0 +1,29 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class ImportMapperUpdateInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("mapper_name") + private String name; + + @Pattern(regexp="^[A-Z]{1,2}$") + @JsonProperty("mapper_inject_type_column") + @NotBlank + private String injectTypeColumn; + + @JsonProperty("mapper_inject_importers") + @NotNull + private List importers = new ArrayList<>(); +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterAddInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterAddInput.java new file mode 100644 index 0000000000..4adb2814be --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterAddInput.java @@ -0,0 +1,25 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class InjectImporterAddInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("inject_importer_type_value") + private String injectTypeValue; + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("inject_importer_injector_contract_id") + private String injectorContractId; + + @JsonProperty("inject_importer_rule_attributes") + private List ruleAttributes = new ArrayList<>(); +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterUpdateInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterUpdateInput.java new file mode 100644 index 0000000000..e5871037ac --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/InjectImporterUpdateInput.java @@ -0,0 +1,28 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class InjectImporterUpdateInput { + + @JsonProperty("inject_importer_id") + private String id; + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("inject_importer_type_value") + private String injectTypeValue; + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("inject_importer_injector_contract_id") + private String injectorContractId; + + @JsonProperty("inject_importer_rule_attributes") + private List ruleAttributes = new ArrayList<>(); +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeAddInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeAddInput.java new file mode 100644 index 0000000000..531ac10a21 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeAddInput.java @@ -0,0 +1,30 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class RuleAttributeAddInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("rule_attribute_name") + private String name; + + @JsonProperty("rule_attribute_columns") + @Schema(nullable = true) + private String columns; + + @JsonProperty("rule_attribute_default_value") + private String defaultValue; + + @JsonProperty("rule_attribute_additional_config") + private Map additionalConfig = new HashMap<>(); + +} diff --git a/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeUpdateInput.java b/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeUpdateInput.java new file mode 100644 index 0000000000..7d7f94a3bc --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/mapper/form/RuleAttributeUpdateInput.java @@ -0,0 +1,33 @@ +package io.openbas.rest.mapper.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class RuleAttributeUpdateInput { + + @JsonProperty("rule_attribute_id") + private String id; + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("rule_attribute_name") + private String name; + + @JsonProperty("rule_attribute_columns") + @Schema(nullable = true) + private String columns; + + @JsonProperty("rule_attribute_default_value") + private String defaultValue; + + @JsonProperty("rule_attribute_additional_config") + private Map additionalConfig = new HashMap<>(); + +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/ScenarioImportApi.java b/openbas-api/src/main/java/io/openbas/rest/scenario/ScenarioImportApi.java new file mode 100644 index 0000000000..3288c6f2ae --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/ScenarioImportApi.java @@ -0,0 +1,71 @@ +package io.openbas.rest.scenario; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.model.Scenario; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.rest.exception.ElementNotFoundException; +import io.openbas.rest.helper.RestBehavior; +import io.openbas.rest.scenario.form.InjectsImportInput; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.service.InjectService; +import io.openbas.service.ScenarioService; +import io.swagger.v3.oas.annotations.Operation; +import jakarta.transaction.Transactional; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotBlank; +import lombok.RequiredArgsConstructor; +import lombok.extern.java.Log; +import org.springframework.security.access.annotation.Secured; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RestController; + +import java.util.UUID; + +import static io.openbas.database.model.User.ROLE_USER; +import static io.openbas.rest.scenario.ScenarioApi.SCENARIO_URI; + +@RestController +@RequiredArgsConstructor +@Log +public class ScenarioImportApi extends RestBehavior { + + private final InjectService injectService; + private final ImportMapperRepository importMapperRepository; + private final ScenarioService scenarioService; + + @PostMapping(SCENARIO_URI + "/{scenarioId}/xls/{importId}/dry") + @Transactional(rollbackOn = Exception.class) + @Operation(summary = "Test the import of injects from an xls file") + @Secured(ROLE_USER) + public ImportTestSummary dryRunImportXLSFile(@PathVariable @NotBlank final String scenarioId, + @PathVariable @NotBlank final String importId, + @Valid @RequestBody final InjectsImportInput input) { + Scenario scenario = scenarioService.scenario(scenarioId); + + // Getting the mapper to use + ImportMapper importMapper = importMapperRepository + .findById(UUID.fromString(input.getImportMapperId())) + .orElseThrow(() -> new ElementNotFoundException(String.format("The import mapper %s was not found", input.getImportMapperId()))); + + return injectService.importInjectIntoScenarioFromXLS(scenario, importMapper, importId, input.getName(), input.getTimezoneOffset(), false); + } + + @PostMapping(SCENARIO_URI + "/{scenarioId}/xls/{importId}/import") + @Transactional(rollbackOn = Exception.class) + @Operation(summary = "Validate and import injects from an xls file") + @Secured(ROLE_USER) + public ImportTestSummary validateImportXLSFile(@PathVariable @NotBlank final String scenarioId, + @PathVariable @NotBlank final String importId, + @Valid @RequestBody final InjectsImportInput input) { + Scenario scenario = scenarioService.scenario(scenarioId); + + // Getting the mapper to use + ImportMapper importMapper = importMapperRepository + .findById(UUID.fromString(input.getImportMapperId())) + .orElseThrow(() -> new ElementNotFoundException(String.format("The import mapper %s was not found", input.getImportMapperId()))); + + return injectService.importInjectIntoScenarioFromXLS(scenario, importMapper, importId, input.getName(), input.getTimezoneOffset(), true); + } +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportInput.java b/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportInput.java new file mode 100644 index 0000000000..21d5af24cb --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportInput.java @@ -0,0 +1,24 @@ +package io.openbas.rest.scenario.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class InjectsImportInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("sheet_name") + private String name; + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("import_mapper_id") + private String importMapperId; + + @NotNull(message = MANDATORY_MESSAGE) + @JsonProperty("timezone_offset") + private Integer timezoneOffset; +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportTestInput.java b/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportTestInput.java new file mode 100644 index 0000000000..77d825a0fc --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/form/InjectsImportTestInput.java @@ -0,0 +1,25 @@ +package io.openbas.rest.scenario.form; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.openbas.rest.mapper.form.ImportMapperAddInput; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import static io.openbas.config.AppConfig.MANDATORY_MESSAGE; + +@Data +public class InjectsImportTestInput { + + @NotBlank(message = MANDATORY_MESSAGE) + @JsonProperty("sheet_name") + private String name; + + @NotNull(message = MANDATORY_MESSAGE) + @JsonProperty("import_mapper") + private ImportMapperAddInput importMapper; + + @NotNull(message = MANDATORY_MESSAGE) + @JsonProperty("timezone_offset") + private Integer timezoneOffset; +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportMessage.java b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportMessage.java new file mode 100644 index 0000000000..fabeb30316 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportMessage.java @@ -0,0 +1,54 @@ +package io.openbas.rest.scenario.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Data +@NoArgsConstructor +public class ImportMessage { + public enum MessageLevel { + CRITICAL, + ERROR, + WARN, + INFO + } + public enum ErrorCode { + NO_POTENTIAL_MATCH_FOUND("no_potential_match_found"), + SEVERAL_MATCHES ("several_matches"), + ABSOLUTE_TIME_WITHOUT_START_DATE ("absolute_time_without_start_date"), + DATE_SET_IN_PAST ("date_set_in_past"), + DATE_SET_IN_FUTURE ("date_set_in_future"), + NO_TEAM_FOUND("no_team_found"), + EXPECTATION_SCORE_UNDEFINED("expectation_score_undefined"); + + public final String code; + + ErrorCode(String code) { + this.code = code; + } + } + + @JsonProperty("message_level") + private MessageLevel messageLevel; + + @JsonProperty("message_code") + private ErrorCode errorCode; + + @JsonProperty("message_params") + private Map params = null; + + public ImportMessage(MessageLevel level, ErrorCode errorCode, Map params) { + this.messageLevel = level; + this.errorCode = errorCode; + this.params = params; + } + + public ImportMessage(MessageLevel level, ErrorCode errorCode) { + this.messageLevel = level; + this.errorCode = errorCode; + } + +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportPostSummary.java b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportPostSummary.java new file mode 100644 index 0000000000..6a60a25afc --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportPostSummary.java @@ -0,0 +1,21 @@ +package io.openbas.rest.scenario.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +import java.util.List; + +@Data +public class ImportPostSummary { + + @JsonProperty("import_id") + @NotBlank + private String importId; + + @JsonProperty("available_sheets") + @NotNull + private List availableSheets; + +} diff --git a/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportTestSummary.java b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportTestSummary.java new file mode 100644 index 0000000000..0a1c25fede --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/rest/scenario/response/ImportTestSummary.java @@ -0,0 +1,27 @@ +package io.openbas.rest.scenario.response; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.openbas.database.model.Inject; +import io.openbas.rest.atomic_testing.form.InjectResultDTO; +import io.openbas.utils.AtomicTestingMapper; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class ImportTestSummary { + + @JsonProperty("import_message") + private List importMessage = new ArrayList<>(); + + @JsonIgnore + private List injects = new ArrayList<>(); + + @JsonProperty("injects") + public List getInjectResults() { + return injects.stream().map(AtomicTestingMapper::toDtoWithTargetResults).toList(); + } + +} diff --git a/openbas-api/src/main/java/io/openbas/service/ImportRow.java b/openbas-api/src/main/java/io/openbas/service/ImportRow.java new file mode 100644 index 0000000000..464ba1c133 --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/service/ImportRow.java @@ -0,0 +1,15 @@ +package io.openbas.service; + +import io.openbas.database.model.Inject; +import io.openbas.rest.scenario.response.ImportMessage; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +public class ImportRow { + private InjectTime injectTime; + private List importMessages = new ArrayList<>(); + private Inject inject; +} diff --git a/openbas-api/src/main/java/io/openbas/service/InjectService.java b/openbas-api/src/main/java/io/openbas/service/InjectService.java index 0753a653e0..e0ba082660 100644 --- a/openbas-api/src/main/java/io/openbas/service/InjectService.java +++ b/openbas-api/src/main/java/io/openbas/service/InjectService.java @@ -1,11 +1,21 @@ package io.openbas.service; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import io.openbas.database.model.*; import io.openbas.database.raw.*; import io.openbas.database.repository.*; +import io.openbas.rest.exception.BadRequestException; +import io.openbas.rest.exception.ElementNotFoundException; import io.openbas.rest.inject.form.InjectUpdateStatusInput; import io.openbas.rest.inject.output.InjectOutput; +import io.openbas.rest.scenario.response.ImportMessage; +import io.openbas.rest.scenario.response.ImportPostSummary; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.utils.InjectUtils; +import jakarta.annotation.Resource; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; import jakarta.persistence.Tuple; @@ -14,72 +24,113 @@ import jakarta.transaction.Transactional; import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; +import lombok.extern.java.Log; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.util.Strings; +import org.apache.poi.ss.usermodel.*; +import org.apache.poi.ss.util.CellReference; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.*; +import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; +import java.time.temporal.ChronoField; +import java.time.temporal.ChronoUnit; +import java.time.temporal.Temporal; import java.util.*; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.stream.StreamSupport; +import static io.openbas.config.SessionHelper.currentUser; import static io.openbas.utils.JpaUtils.createJoinArrayAggOnId; import static io.openbas.utils.JpaUtils.createLeftJoin; import static java.time.Instant.now; @RequiredArgsConstructor @Service +@Log public class InjectService { - private final InjectRepository injectRepository; - private final InjectDocumentRepository injectDocumentRepository; - private final InjectExpectationRepository injectExpectationRepository; - private final AssetRepository assetRepository; - private final AssetGroupRepository assetGroupRepository; - private final TeamRepository teamRepository; - - @PersistenceContext - private EntityManager entityManager; - - public void cleanInjectsDocExercise(String exerciseId, String documentId) { - // Delete document from all exercise injects - List exerciseInjects = injectRepository.findAllForExerciseAndDoc(exerciseId, documentId); - List updatedInjects = exerciseInjects.stream().flatMap(inject -> { - @SuppressWarnings("UnnecessaryLocalVariable") - Stream filterDocuments = inject.getDocuments().stream() - .filter(document -> document.getDocument().getId().equals(documentId)); - return filterDocuments; - }).toList(); - injectDocumentRepository.deleteAll(updatedInjects); - } + private final InjectRepository injectRepository; + private final InjectDocumentRepository injectDocumentRepository; + private final InjectExpectationRepository injectExpectationRepository; + private final AssetRepository assetRepository; + private final AssetGroupRepository assetGroupRepository; + private final ScenarioTeamUserRepository scenarioTeamUserRepository; + private final TeamRepository teamRepository; + private final UserRepository userRepository; + private final ScenarioService scenarioService; - public void cleanInjectsDocScenario(String scenarioId, String documentId) { - // Delete document from all scenario injects - List scenarioInjects = injectRepository.findAllForScenarioAndDoc(scenarioId, documentId); - List updatedInjects = scenarioInjects.stream().flatMap(inject -> { - @SuppressWarnings("UnnecessaryLocalVariable") - Stream filterDocuments = inject.getDocuments().stream() - .filter(document -> document.getDocument().getId().equals(documentId)); - return filterDocuments; - }).toList(); - injectDocumentRepository.deleteAll(updatedInjects); - } + private final List importReservedField = List.of("description", "title", "trigger_time"); - @Transactional(rollbackOn = Exception.class) - public Inject updateInjectStatus(String injectId, InjectUpdateStatusInput input) { - Inject inject = injectRepository.findById(injectId).orElseThrow(); - // build status - InjectStatus injectStatus = new InjectStatus(); - injectStatus.setInject(inject); - injectStatus.setTrackingSentDate(now()); - injectStatus.setName(ExecutionStatus.valueOf(input.getStatus())); - injectStatus.setTrackingTotalExecutionTime(0L); - // Save status for inject - inject.setStatus(injectStatus); - return injectRepository.save(inject); - } + @Resource + protected ObjectMapper mapper; + @PersistenceContext + private EntityManager entityManager; + + final Pattern relativeDayPattern = Pattern.compile("^.*[DJ]([+\\-]?[0-9]*)(.*)$"); + final Pattern relativeHourPattern = Pattern.compile("^.*[HT]([+\\-]?[0-9]*).*$"); + final Pattern relativeMinutePattern = Pattern.compile("^.*[M]([+\\-]?[0-9]*).*$"); + + final String pathSeparator = FileSystems.getDefault().getSeparator(); + + final int FILE_STORAGE_DURATION = 60; + + public void cleanInjectsDocExercise(String exerciseId, String documentId) { + // Delete document from all exercise injects + List exerciseInjects = injectRepository.findAllForExerciseAndDoc(exerciseId, documentId); + List updatedInjects = exerciseInjects.stream().flatMap(inject -> { + @SuppressWarnings("UnnecessaryLocalVariable") + Stream filterDocuments = inject.getDocuments().stream() + .filter(document -> document.getDocument().getId().equals(documentId)); + return filterDocuments; + }).toList(); + injectDocumentRepository.deleteAll(updatedInjects); + } + + public void cleanInjectsDocScenario(String scenarioId, String documentId) { + // Delete document from all scenario injects + List scenarioInjects = injectRepository.findAllForScenarioAndDoc(scenarioId, documentId); + List updatedInjects = scenarioInjects.stream().flatMap(inject -> { + @SuppressWarnings("UnnecessaryLocalVariable") + Stream filterDocuments = inject.getDocuments().stream() + .filter(document -> document.getDocument().getId().equals(documentId)); + return filterDocuments; + }).toList(); + injectDocumentRepository.deleteAll(updatedInjects); + } - public List injects(Specification specification) { - CriteriaBuilder cb = this.entityManager.getCriteriaBuilder(); + @Transactional(rollbackOn = Exception.class) + public Inject updateInjectStatus(String injectId, InjectUpdateStatusInput input) { + Inject inject = injectRepository.findById(injectId).orElseThrow(); + // build status + InjectStatus injectStatus = new InjectStatus(); + injectStatus.setInject(inject); + injectStatus.setTrackingSentDate(now()); + injectStatus.setName(ExecutionStatus.valueOf(input.getStatus())); + injectStatus.setTrackingTotalExecutionTime(0L); + // Save status for inject + inject.setStatus(injectStatus); + return injectRepository.save(inject); + } + + public List injects(Specification specification) { + CriteriaBuilder cb = this.entityManager.getCriteriaBuilder(); CriteriaQuery cq = cb.createTupleQuery(); Root injectRoot = cq.from(Inject.class); @@ -103,104 +154,793 @@ public List injects(Specification specification) { return execInject(query); } - // -- CRITERIA BUILDER -- - - private void selectForInject(CriteriaBuilder cb, CriteriaQuery cq, Root injectRoot) { - // Joins - Join injectExerciseJoin = createLeftJoin(injectRoot, "exercise"); - Join injectScenarioJoin = createLeftJoin(injectRoot, "scenario"); - Join injectorContractJoin = createLeftJoin(injectRoot, "injectorContract"); - Join injectorJoin = injectorContractJoin.join("injector", JoinType.LEFT); - // Array aggregations - Expression tagIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "tags"); - Expression teamIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "teams"); - Expression assetIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "assets"); - Expression assetGroupIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "assetGroups"); - - // SELECT - cq.multiselect( - injectRoot.get("id").alias("inject_id"), - injectRoot.get("title").alias("inject_title"), - injectRoot.get("enabled").alias("inject_enabled"), - injectRoot.get("content").alias("inject_content"), - injectRoot.get("allTeams").alias("inject_all_teams"), - injectExerciseJoin.get("id").alias("inject_exercise"), - injectScenarioJoin.get("id").alias("inject_scenario"), - injectRoot.get("dependsDuration").alias("inject_depends_duration"), - injectorContractJoin.alias("inject_injector_contract"), - tagIdsExpression.alias("inject_tags"), - teamIdsExpression.alias("inject_teams"), - assetIdsExpression.alias("inject_assets"), - assetGroupIdsExpression.alias("inject_asset_groups"), - injectorJoin.get("type").alias("inject_type") - ).distinct(true); - - // GROUP BY - cq.groupBy(Arrays.asList( - injectRoot.get("id"), - injectExerciseJoin.get("id"), - injectScenarioJoin.get("id"), - injectorContractJoin.get("id"), - injectorJoin.get("id") - )); - } + /** + * Create inject programmatically based on rawInject, rawInjectExpectation, rawAsset, rawAssetGroup, rawTeam + */ + public Map mapOfInjects(@NotNull final List injectIds) { + List listOfInjects = new ArrayList<>(); - private List execInject(TypedQuery query) { - return query.getResultList() - .stream() - .map(tuple -> new InjectOutput( - tuple.get("inject_id", String.class), - tuple.get("inject_title", String.class), - tuple.get("inject_enabled", Boolean.class), - tuple.get("inject_content", ObjectNode.class), - tuple.get("inject_all_teams", Boolean.class), - tuple.get("inject_exercise", String.class), - tuple.get("inject_scenario", String.class), - tuple.get("inject_depends_duration", Long.class), - tuple.get("inject_injector_contract", InjectorContract.class), - tuple.get("inject_tags", String[].class), - tuple.get("inject_teams", String[].class), - tuple.get("inject_assets", String[].class), - tuple.get("inject_asset_groups", String[].class), - tuple.get("inject_type", String.class) - )) - .toList(); - } + List listOfRawInjects = this.injectRepository.findRawByIds(injectIds); + // From the list of injects, we get all the inject expectationsIds that we then get + // and put into a map with the expections ids as key + Map mapOfInjectsExpectations = mapOfInjectsExpectations(listOfRawInjects); - // -- TEST -- + // We get the asset groups from the injects AND the injects expectations as those can also have asset groups + // We then make a map out of it for faster access + Map mapOfAssetGroups = mapOfAssetGroups(listOfRawInjects, mapOfInjectsExpectations.values()); - /** - * Create inject programmatically based on rawInject, rawInjectExpectation, rawAsset, rawAssetGroup, rawTeam - */ - public Map mapOfInjects(@NotNull final List injectIds) { - List listOfInjects = new ArrayList<>(); - - List listOfRawInjects = this.injectRepository.findRawByIds(injectIds); - // From the list of injects, we get all the inject expectationsIds that we then get - // and put into a map with the expections ids as key - Map mapOfInjectsExpectations = mapOfInjectsExpectations(listOfRawInjects); - - // We get the asset groups from the injects AND the injects expectations as those can also have asset groups - // We then make a map out of it for faster access - Map mapOfAssetGroups = mapOfAssetGroups(listOfRawInjects, mapOfInjectsExpectations.values()); - - // We get all the assets that are - // 1 - linked to an inject - // 2 - linked to an asset group linked to an inject - // 3 - linked to an inject expectation - // 4 - linked to an asset group linked to an inject expectations - // We then make a map out of it - Map mapOfAssets = mapOfAssets(listOfRawInjects, mapOfInjectsExpectations, mapOfAssetGroups); - - // We get all the teams that are linked to an inject or an asset group - // Then we make a map out of it for faster access - Map mapOfRawTeams = mapOfRawTeams(listOfRawInjects, mapOfInjectsExpectations); - - // Once we have all of this, we create an Inject for each InjectRaw that we have using all the Raw objects we got - // Then we make a map out of it for faster access - listOfRawInjects.stream().map((inject) -> Inject.fromRawInject(inject, mapOfRawTeams, mapOfInjectsExpectations, mapOfAssetGroups, mapOfAssets)).forEach(listOfInjects::add); - return listOfInjects.stream().collect(Collectors.toMap(Inject::getId, Function.identity())); - } + // We get all the assets that are + // 1 - linked to an inject + // 2 - linked to an asset group linked to an inject + // 3 - linked to an inject expectation + // 4 - linked to an asset group linked to an inject expectations + // We then make a map out of it + Map mapOfAssets = mapOfAssets(listOfRawInjects, mapOfInjectsExpectations, mapOfAssetGroups); + + // We get all the teams that are linked to an inject or an asset group + // Then we make a map out of it for faster access + Map mapOfRawTeams = mapOfRawTeams(listOfRawInjects, mapOfInjectsExpectations); + + // Once we have all of this, we create an Inject for each InjectRaw that we have using all the Raw objects we got + // Then we make a map out of it for faster access + listOfRawInjects.stream().map((inject) -> Inject.fromRawInject(inject, mapOfRawTeams, mapOfInjectsExpectations, mapOfAssetGroups, mapOfAssets)).forEach(listOfInjects::add); + return listOfInjects.stream().collect(Collectors.toMap(Inject::getId, Function.identity())); + } + + /** + * Store an xls file for ulterior import. The file will be deleted on exit. + * @param file + * @return + */ + public ImportPostSummary storeXlsFileForImport(MultipartFile file) { + ImportPostSummary result = new ImportPostSummary(); + result.setAvailableSheets(new ArrayList<>()); + // Generating an UUID for identifying the file + String fileID = UUID.randomUUID().toString(); + result.setImportId(fileID); + try { + // We're opening the file and listing the names of the sheets + Workbook workbook = WorkbookFactory.create(file.getInputStream()); + for (int i = 0; i < workbook.getNumberOfSheets(); i++) { + result.getAvailableSheets().add(workbook.getSheetName(i)); + } + // Writing the file in a temp dir + Path tempDir = Files.createDirectory(Path.of(System.getProperty("java.io.tmpdir"), fileID)); + Path tempFile = Files.createTempFile(tempDir, null, "." + FilenameUtils.getExtension(file.getOriginalFilename())); + Files.write(tempFile, file.getBytes()); + + CompletableFuture.delayedExecutor(FILE_STORAGE_DURATION, TimeUnit.MINUTES).execute(() -> { + tempFile.toFile().delete(); + tempDir.toFile().delete(); + }); + + // We're making sure the files are deleted when the backend restart + tempDir.toFile().deleteOnExit(); + tempFile.toFile().deleteOnExit(); + } catch (Exception ex) { + log.severe("Error while importing an xls file"); + log.severe(Arrays.toString(ex.getStackTrace())); + throw new BadRequestException("File seems to be corrupt"); + } + + return result; + } + + public ImportTestSummary importInjectIntoScenarioFromXLS(Scenario scenario, ImportMapper importMapper, String importId, String sheetName, int timezoneOffset, boolean saveAll) { + // We call the inject service to get the injects to create as well as messages on how things went + ImportTestSummary importTestSummary = importXls(importId, scenario, importMapper, sheetName, timezoneOffset); + Optional hasCritical = importTestSummary.getImportMessage().stream() + .filter(importMessage -> importMessage.getMessageLevel() == ImportMessage.MessageLevel.CRITICAL) + .findAny(); + if(hasCritical.isPresent()) { + // If there are critical errors, we do not save and we + // empty the list of injects, we just keep the messages + importTestSummary.setInjects(new ArrayList<>()); + } else if(saveAll) { + Iterable newInjects = injectRepository.saveAll(importTestSummary.getInjects()); + newInjects.forEach(inject -> { + scenario.getInjects().add(inject); + inject.getTeams().forEach(team -> { + if (!scenario.getTeams().contains(team)) { + scenario.getTeams().add(team); + } + }); + inject.getTeams().forEach(team -> team.getUsers().forEach(user -> { + if(!scenario.getTeamUsers().contains(user)) { + ScenarioTeamUser scenarioTeamUser = new ScenarioTeamUser(); + scenarioTeamUser.setScenario(scenario); + scenarioTeamUser.setTeam(team); + scenarioTeamUser.setUser(user); + scenarioTeamUserRepository.save(scenarioTeamUser); + scenario.getTeamUsers().add(scenarioTeamUser); + } + })); + }); + scenarioService.updateScenario(scenario); + } + + return importTestSummary; + } + + private ImportTestSummary importXls(String importId, Scenario scenario, ImportMapper importMapper, String sheetName, int timezoneOffset) { + ImportTestSummary importTestSummary = new ImportTestSummary(); + + try { + // We open the previously saved file + String tmpdir = System.getProperty("java.io.tmpdir"); + java.nio.file.Path file = Files.list(Path.of(tmpdir, pathSeparator, importId, pathSeparator)).findFirst().orElseThrow(); + + // We open the file and convert it to an apache POI object + InputStream xlsFile = Files.newInputStream(file); + Workbook workbook = WorkbookFactory.create(xlsFile); + Sheet selectedSheet = workbook.getSheet(sheetName); + + Map mapInstantByRowIndex = new HashMap<>(); + + // For performance reasons, we compile the pattern of the Inject Importers only once + Map mapPatternByInjectImport = importMapper + .getInjectImporters().stream().collect( + Collectors.toMap(InjectImporter::getId, + injectImporter -> Pattern.compile(injectImporter.getImportTypeValue()) + )); + + // We also get the list of teams into a map to be able to get them easily later on + // First, all the teams that are non-contextual + Map mapTeamByName = + StreamSupport.stream(teamRepository.findAll().spliterator(), false) + .filter(team -> !team.getContextual()) + .collect(Collectors.toMap(Team::getName, Function.identity(), (first, second) -> first)); + + // Then we add the contextual teams of the scenario + mapTeamByName.putAll( + scenario.getTeams().stream() + .filter(Team::getContextual) + .collect(Collectors.toMap(Team::getName, Function.identity(), (first, second) -> first)) + ); + + ZoneOffset zoneOffset = ZoneOffset.ofTotalSeconds(timezoneOffset * 60); + + // For each rows of the selected sheet + selectedSheet.rowIterator().forEachRemaining(row -> { + ImportRow rowSummary = importRow(row, importMapper, scenario, mapPatternByInjectImport, mapTeamByName, + zoneOffset); + importTestSummary.getImportMessage().addAll(rowSummary.getImportMessages()); + if(rowSummary.getInject() != null) { + importTestSummary.getInjects().add(rowSummary.getInject()); + } + if(rowSummary.getInjectTime() != null) { + mapInstantByRowIndex.put(row.getRowNum(), rowSummary.getInjectTime()); + } + }); + + // Now that we did our first pass, we do another one real quick to find out + // the date relative to each others + importTestSummary.getImportMessage().addAll(updateInjectDates(mapInstantByRowIndex)); + + // We get the earliest date + Optional earliestDate = mapInstantByRowIndex.values().stream() + .map(InjectTime::getDate) + .filter(Objects::nonNull) + .min(Comparator.naturalOrder()); + + // If there is one, we update the scenario date + earliestDate.ifPresent(date -> { + ZonedDateTime zonedDateTime = date.atZone(ZoneId.of("UTC")); + Instant dayOfStart = ZonedDateTime.of( + zonedDateTime.getYear(), zonedDateTime.getMonthValue(), zonedDateTime.getDayOfMonth(), + 0, 0, 0, 0, ZoneId.of("UTC")).toInstant(); + scenario.setRecurrenceStart(dayOfStart); + scenario.setRecurrence("0 " + zonedDateTime.getMinute() + " " + zonedDateTime.getHour() + " * * *"); // Every day now + 1 hour + scenario.setRecurrenceEnd(dayOfStart.plus(1, ChronoUnit.DAYS)); + }); + } catch (IOException ex) { + log.severe("Error while importing an xls file"); + log.severe(Arrays.toString(ex.getStackTrace())); + throw new BadRequestException(); + } + + // Sorting by the order of the enum declaration to have error messages first, then warn and then info + importTestSummary.getImportMessage().sort(Comparator.comparing(ImportMessage::getMessageLevel)); + + return importTestSummary; + } + + private ImportRow importRow(Row row, ImportMapper importMapper, Scenario scenario, + Map mapPatternByInjectImport, Map mapTeamByName, + ZoneOffset timezoneOffset) { + ImportRow importTestSummary = new ImportRow(); + // The column that differenciate the importer is the same for all so we get it right now + int colTypeIdx = CellReference.convertColStringToIndex(importMapper.getInjectTypeColumn()); + + //If the row is completely empty, we ignore it altogether and do not send a warn message + if (InjectUtils.checkIfRowIsEmpty(row)) { + return importTestSummary; + } + // First of all, we get the value of the differenciation cell + Cell typeCell = row.getCell(colTypeIdx); + if (typeCell == null) { + // If there are no values, we add an info message so they know there is a potential issue here + importTestSummary.getImportMessages().add( + new ImportMessage(ImportMessage.MessageLevel.INFO, + ImportMessage.ErrorCode.NO_POTENTIAL_MATCH_FOUND, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum())) + ) + ); + return importTestSummary; + } + + // We find the matching importers on the inject + List matchingInjectImporters = importMapper.getInjectImporters().stream() + .filter(injectImporter -> { + Matcher matcher = mapPatternByInjectImport.get(injectImporter.getId()) + .matcher(getValueAsString(row, importMapper.getInjectTypeColumn())); + return matcher.find(); + }).toList(); + + // If there are no match, we add a message for the user and we go to the next row + if (matchingInjectImporters.isEmpty()) { + importTestSummary.getImportMessages().add( + new ImportMessage(ImportMessage.MessageLevel.INFO, + ImportMessage.ErrorCode.NO_POTENTIAL_MATCH_FOUND, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum())) + ) + ); + return importTestSummary; + } + + // If there are more than one match, we add a message for the user and use the first match + if (matchingInjectImporters.size() > 1) { + String listMatchers = matchingInjectImporters.stream().map(InjectImporter::getImportTypeValue).collect(Collectors.joining(", ")); + importTestSummary.getImportMessages().add( + new ImportMessage(ImportMessage.MessageLevel.WARN, + ImportMessage.ErrorCode.SEVERAL_MATCHES, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum()), + "possible_matches", listMatchers) + ) + ); + return importTestSummary; + } + + InjectImporter matchingInjectImporter = matchingInjectImporters.get(0); + InjectorContract injectorContract = matchingInjectImporter.getInjectorContract(); + + // Creating the inject + Inject inject = new Inject(); + inject.setDependsDuration(0L); + + // Adding the description + setAttributeValue(row, matchingInjectImporter, "description", inject::setDescription); + + // Adding the title + setAttributeValue(row, matchingInjectImporter, "title", inject::setTitle); + + // Adding the trigger time + RuleAttribute triggerTimeRuleAttribute = matchingInjectImporter.getRuleAttributes().stream() + .filter(ruleAttribute -> ruleAttribute.getName().equals("trigger_time")) + .findFirst().orElseGet(() -> { + RuleAttribute ruleAttributeDefault = new RuleAttribute(); + ruleAttributeDefault.setAdditionalConfig( + Map.of("timePattern", "") + ); + return ruleAttributeDefault; + }); + + String timePattern = triggerTimeRuleAttribute.getAdditionalConfig() + .get("timePattern"); + String dateAsString = Strings.EMPTY; + if(triggerTimeRuleAttribute.getColumns() != null) { + dateAsString = Arrays.stream(triggerTimeRuleAttribute.getColumns().split("\\+")) + .map(column -> getDateAsStringFromCell(row, column)) + .collect(Collectors.joining()); + } + if (dateAsString.isBlank()) { + dateAsString = triggerTimeRuleAttribute.getDefaultValue(); + } + + Matcher relativeDayMatcher = relativeDayPattern.matcher(dateAsString); + Matcher relativeHourMatcher = relativeHourPattern.matcher(dateAsString); + Matcher relativeMinuteMatcher = relativeMinutePattern.matcher(dateAsString); + + + boolean relativeDays = relativeDayMatcher.matches(); + boolean relativeHour = relativeHourMatcher.matches(); + boolean relativeMinute = relativeMinuteMatcher.matches(); + + InjectTime injectTime = new InjectTime(); + injectTime.setUnformattedDate(dateAsString); + injectTime.setLinkedInject(inject); + + Temporal dateTime = getInjectDate(injectTime, timePattern); + + if (dateTime == null) { + injectTime.setRelativeDay(relativeDays); + if(relativeDays && relativeDayMatcher.groupCount() > 0 && !relativeDayMatcher.group(1).isBlank()) { + injectTime.setRelativeDayNumber(Integer.parseInt(relativeDayMatcher.group(1))); + } + injectTime.setRelativeHour(relativeHour); + if(relativeHour && relativeHourMatcher.groupCount() > 0 && !relativeHourMatcher.group(1).isBlank()) { + injectTime.setRelativeHourNumber(Integer.parseInt(relativeHourMatcher.group(1))); + } + injectTime.setRelativeMinute(relativeMinute); + if(relativeMinute && relativeMinuteMatcher.groupCount() > 0 && !relativeMinuteMatcher.group(1).isBlank()) { + injectTime.setRelativeMinuteNumber(Integer.parseInt(relativeMinuteMatcher.group(1))); + } + + // Special case : a mix of relative day and absolute hour + if(relativeDays && relativeDayMatcher.groupCount() > 1 && !relativeDayMatcher.group(2).isBlank()) { + Temporal date = null; + try { + date = LocalTime.parse(relativeDayMatcher.group(2).trim(), injectTime.getFormatter()); + } catch (DateTimeParseException firstException) { + try { + date = LocalTime.parse(relativeDayMatcher.group(2).trim(), DateTimeFormatter.ISO_TIME); + } catch (DateTimeParseException exception) { + // This is a "probably" a relative date + } + } + if(date != null) { + if(scenario.getRecurrenceStart() != null) { + injectTime.setDate( + scenario.getRecurrenceStart() + .atZone(timezoneOffset).toLocalDateTime() + .withHour(date.get(ChronoField.HOUR_OF_DAY)) + .withMinute(date.get(ChronoField.MINUTE_OF_HOUR)) + .toInstant(timezoneOffset) + ); + } else { + importTestSummary.getImportMessages().add( + new ImportMessage(ImportMessage.MessageLevel.CRITICAL, + ImportMessage.ErrorCode.ABSOLUTE_TIME_WITHOUT_START_DATE, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum())) + ) + ); + } + } + } + } + injectTime.setSpecifyDays(relativeDays || injectTime.getFormatter().equals(DateTimeFormatter.ISO_DATE_TIME)); + + // We get the absolute dates available on our first pass + if(!relativeDays && !relativeHour && !relativeMinute && dateTime != null) { + if (dateTime instanceof LocalDateTime) { + Instant injectDate = Instant.ofEpochSecond( + ((LocalDateTime)dateTime).toEpochSecond(timezoneOffset)); + injectTime.setDate(injectDate); + } else if (dateTime instanceof LocalTime) { + if(scenario.getRecurrenceStart() != null) { + injectTime.setDate(scenario.getRecurrenceStart() + .atZone(timezoneOffset) + .withHour(((LocalTime) dateTime).getHour()) + .withMinute(((LocalTime) dateTime).getMinute()) + .toInstant()); + } else { + importTestSummary.getImportMessages().add( + new ImportMessage(ImportMessage.MessageLevel.CRITICAL, + ImportMessage.ErrorCode.ABSOLUTE_TIME_WITHOUT_START_DATE, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum())) + ) + ); + return importTestSummary; + } + } + } + + // Initializing the content with a root node + ObjectMapper mapper = new ObjectMapper(); + inject.setContent(mapper.createObjectNode()); + + // Once it's done, we set the injectorContract + inject.setInjectorContract(injectorContract); + + // So far, we only support one expectation + AtomicReference expectation = new AtomicReference<>(); + + // For each rule attributes of the importer + matchingInjectImporter.getRuleAttributes().forEach(ruleAttribute -> { + importTestSummary.getImportMessages().addAll( + addFields(inject, ruleAttribute, + row, mapTeamByName, expectation, importMapper)); + }); + // This is by default at false + inject.setAllTeams(false); + // The user is the one doing the import + inject.setUser(userRepository.findById(currentUser().getId()).orElseThrow()); + // No exercise yet + inject.setExercise(null); + // No dependencies + inject.setDependsOn(null); + + if(expectation.get() != null) { + // We set the expectation + ArrayNode expectationsNode = mapper.createArrayNode(); + ObjectNode expectationNode = mapper.createObjectNode(); + expectationNode.put("expectation_description", expectation.get().getDescription()); + expectationNode.put("expectation_name", expectation.get().getName()); + expectationNode.put("expectation_score", expectation.get().getExpectedScore()); + expectationNode.put("expectation_type", expectation.get().getType().name()); + expectationNode.put("expectation_expectation_group", false); + expectationsNode.add(expectationNode); + inject.getContent().set("expectations", expectationsNode); + } + + // We set the scenario + inject.setScenario(scenario); + + importTestSummary.setInject(inject); + importTestSummary.setInjectTime(injectTime); + return importTestSummary; + } + + private void setAttributeValue(Row row, InjectImporter matchingInjectImporter, String attributeName, Consumer setter) { + RuleAttribute ruleAttribute = matchingInjectImporter.getRuleAttributes().stream() + .filter(attr -> attr.getName().equals(attributeName)) + .findFirst() + .orElseThrow(ElementNotFoundException::new); + + if(ruleAttribute.getColumns() != null) { + int colIndex = CellReference.convertColStringToIndex(ruleAttribute.getColumns()); + if(colIndex == -1) return; + Cell valueCell = row.getCell(colIndex); + + if (valueCell == null) { + setter.accept(ruleAttribute.getDefaultValue()); + } else { + String cellValue = getValueAsString(row, ruleAttribute.getColumns()); + setter.accept(cellValue.isBlank() ? ruleAttribute.getDefaultValue() : cellValue); + } + } + + } + + private List addFields(Inject inject, RuleAttribute ruleAttribute, + Row row, Map mapTeamByName, + AtomicReference expectation, + ImportMapper importMapper) { + // If it's a reserved field, it's already taken care of + if(importReservedField.contains(ruleAttribute.getName())) { + return Collections.emptyList(); + } + + // For ease of use, we create a map of the available keys for the injector + Map mapFieldByKey = + StreamSupport.stream((inject.getInjectorContract().getConvertedContent().get("fields") + .spliterator()), false) + .collect(Collectors.toMap(jsonNode -> jsonNode.get("key").asText(), Function.identity())); + + // Otherwise, the default type is text but it can be overriden + String type = "text"; + if (mapFieldByKey.get(ruleAttribute.getName()) != null) { + type = mapFieldByKey.get(ruleAttribute.getName()).get("type").asText(); + } else if(ruleAttribute.getName().startsWith("expectation")) { + type = "expectation"; + } + switch (type) { + case "text": + case "textarea": + // If text, we get the columns, split by "+" if there is a concatenation of columns + // and then joins the result of the cells + String columnValue = Strings.EMPTY; + if(ruleAttribute.getColumns() != null) { + columnValue = Arrays.stream(ruleAttribute.getColumns().split("\\+")) + .map(column -> getValueAsString(row, column)) + .collect(Collectors.joining()); + } + if (columnValue.isBlank()) { + inject.getContent().put(ruleAttribute.getDefaultValue(), columnValue); + } else { + inject.getContent().put(ruleAttribute.getName(), columnValue); + } + break; + case "team": + // If the rule type is on a team field, we split by "+" if there is a concatenation of columns + // and then joins the result, split again by "," and use the list of results to get the teams by their name + + List columnValues = new ArrayList<>(); + if(ruleAttribute.getColumns() != null) { + columnValues = Arrays.stream(Arrays.stream(ruleAttribute.getColumns().split("\\+")) + .map(column -> getValueAsString(row, column)) + .collect(Collectors.joining(",")) + .split(",")) + .toList(); + } + if (columnValues.isEmpty() || columnValues.stream().allMatch(String::isEmpty)) { + List defaultValues = Arrays.stream(ruleAttribute.getDefaultValue().split(",")).toList(); + inject.getTeams().addAll(mapTeamByName.entrySet().stream() + .filter(nameTeamEntry -> defaultValues.contains(nameTeamEntry.getKey())) + .map(Map.Entry::getValue) + .toList() + ); + } else { + List importMessages = new ArrayList<>(); + columnValues.forEach(teamName -> { + if(mapTeamByName.containsKey(teamName)) { + inject.getTeams().add(mapTeamByName.get(teamName)); + } else { + // The team does not exist, we create a new one + Team team = new Team(); + team.setName(teamName); + team.setContextual(true); + teamRepository.save(team); + mapTeamByName.put(team.getName(), team); + inject.getTeams().add(team); + + // We aldo add a message so the user knows there was a new team created + importMessages.add(new ImportMessage(ImportMessage.MessageLevel.WARN, + ImportMessage.ErrorCode.NO_TEAM_FOUND, + Map.of("column_type_num", importMapper.getInjectTypeColumn(), + "row_num", String.valueOf(row.getRowNum()), + "team_name", teamName) + )); + } + }); + if(!importMessages.isEmpty()) { + return importMessages; + } + } + break; + case "expectation": + // If the rule type is of an expectation, + if (expectation.get() == null) { + expectation.set(new InjectExpectation()); + expectation.get().setType(InjectExpectation.EXPECTATION_TYPE.MANUAL); + } + if (ruleAttribute.getName().contains("_")) { + if("score".equals(ruleAttribute.getName().split("_")[1])) { + if(ruleAttribute.getColumns() != null) { + List columns = Arrays.stream(ruleAttribute.getColumns().split("\\+")).toList(); + if(columns.stream().allMatch(s -> row.getCell(CellReference.convertColStringToIndex(s)).getCellType()== CellType.NUMERIC)) { + Double columnValueExpectation = columns.stream() + .map(column -> getValueAsDouble(row, column)) + .reduce(0.0, Double::sum); + expectation.get().setExpectedScore(columnValueExpectation.intValue()); + } else { + try { + expectation.get().setExpectedScore(Integer.parseInt(ruleAttribute.getDefaultValue())); + } catch (NumberFormatException exception) { + List importMessages = new ArrayList<>(); + importMessages.add(new ImportMessage(ImportMessage.MessageLevel.WARN, + ImportMessage.ErrorCode.EXPECTATION_SCORE_UNDEFINED, + Map.of("column_type_num", String.join(", ", columns), + "row_num", String.valueOf(row.getRowNum())) + )); + return importMessages; + } + } + } else { + expectation.get().setExpectedScore(Integer.parseInt(ruleAttribute.getDefaultValue())); + } + } else if ("name".equals(ruleAttribute.getName().split("_")[1])) { + if(ruleAttribute.getColumns() != null) { + String columnValueExpectation = Arrays.stream(ruleAttribute.getColumns().split("\\+")) + .map(column -> getValueAsString(row, column)) + .collect(Collectors.joining()); + expectation.get().setName(columnValueExpectation.isBlank() ? ruleAttribute.getDefaultValue() : columnValueExpectation); + } else { + expectation.get().setName(ruleAttribute.getDefaultValue()); + } + } else if ("description".equals(ruleAttribute.getName().split("_")[1])) { + if(ruleAttribute.getColumns() != null) { + String columnValueExpectation = Arrays.stream(ruleAttribute.getColumns().split("\\+")) + .map(column -> getValueAsString(row, column)) + .collect(Collectors.joining()); + expectation.get().setDescription(columnValueExpectation.isBlank() ? ruleAttribute.getDefaultValue() : columnValueExpectation); + } else { + expectation.get().setDescription(ruleAttribute.getDefaultValue()); + } + } + } + + break; + default: + throw new UnsupportedOperationException(); + } + return Collections.emptyList(); + } + + private Temporal getInjectDate(InjectTime injectTime, String timePattern) { + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME; + if(timePattern != null && !timePattern.isEmpty()) { + dateTimeFormatter = DateTimeFormatter.ofPattern(timePattern); + try { + return LocalDateTime.parse(injectTime.getUnformattedDate(), dateTimeFormatter); + } catch (DateTimeParseException firstException) { + try { + return LocalTime.parse(injectTime.getUnformattedDate(), dateTimeFormatter); + } catch (DateTimeParseException exception) { + // This is a "probably" a relative date + } + } + } else { + try { + return LocalDateTime.parse(injectTime.getUnformattedDate(), dateTimeFormatter); + } catch (DateTimeParseException firstException) { + // The date is not in ISO_DATE_TIME. Trying just the ISO_TIME + dateTimeFormatter = DateTimeFormatter.ISO_TIME; + try { + return LocalDateTime.parse(injectTime.getUnformattedDate(), dateTimeFormatter); + } catch (DateTimeParseException secondException) { + // Neither ISO_DATE_TIME nor ISO_TIME + } + } + } + injectTime.setFormatter(dateTimeFormatter); + return null; + } + + private List updateInjectDates(Map mapInstantByRowIndex) { + List importMessages = new ArrayList<>(); + // First of all, are there any absolute date + boolean allDatesAreAbsolute = mapInstantByRowIndex.values().stream().noneMatch( + injectTime -> injectTime.getDate() == null || injectTime.isRelativeDay() || injectTime.isRelativeHour() || injectTime.isRelativeMinute() + ); + boolean allDatesAreRelative = mapInstantByRowIndex.values().stream().allMatch( + injectTime -> injectTime.getDate() == null && (injectTime.isRelativeDay() || injectTime.isRelativeHour() || injectTime.isRelativeMinute()) + ); + + if(allDatesAreAbsolute) { + processDateToAbsolute(mapInstantByRowIndex); + } else if (allDatesAreRelative) { + // All is relative and we just need to set depends relative to each others + // First of all, we find the minimal relative number of days and hour + int earliestDay = mapInstantByRowIndex.values().stream() + .min(Comparator.comparing(InjectTime::getRelativeDayNumber)) + .map(InjectTime::getRelativeDayNumber).orElse(0); + int earliestHourOfThatDay = mapInstantByRowIndex.values().stream() + .filter(injectTime -> injectTime.getRelativeDayNumber() == earliestDay) + .min(Comparator.comparing(InjectTime::getRelativeHourNumber)) + .map(InjectTime::getRelativeHourNumber).orElse(0); + int earliestMinuteOfThatHour = mapInstantByRowIndex.values().stream() + .filter(injectTime -> injectTime.getRelativeDayNumber() == earliestDay + && injectTime.getRelativeHourNumber() == earliestHourOfThatDay) + .min(Comparator.comparing(InjectTime::getRelativeMinuteNumber)) + .map(InjectTime::getRelativeMinuteNumber).orElse(0); + long offsetAsMinutes = (((earliestDay * 24L) + earliestHourOfThatDay) * 60 + earliestMinuteOfThatHour) * -1; + mapInstantByRowIndex.values().stream().filter(InjectTime::isRelativeDay) + .forEach(injectTime -> { + long injectTimeAsMinutes = + (((injectTime.getRelativeDayNumber() * 24L) + injectTime.getRelativeHourNumber()) * 60) + + injectTime.getRelativeMinuteNumber() + offsetAsMinutes; + injectTime.getLinkedInject().setDependsDuration(injectTimeAsMinutes * 60); + }); + } else { + // Worst case scenario : there is a mix of relative and absolute dates + // We will need to resolve this row by row in the order they are in the import file + Optional> sortedInstantMap = + mapInstantByRowIndex.entrySet().stream().min(Map.Entry.comparingByKey()); + int firstRow; + if(sortedInstantMap.isPresent()) { + firstRow = sortedInstantMap.get().getKey(); + } else { + firstRow = 0; + } + mapInstantByRowIndex.entrySet().stream().sorted(Map.Entry.comparingByKey()).forEachOrdered( + integerInjectTimeEntry -> { + InjectTime injectTime = integerInjectTimeEntry.getValue(); + + if(injectTime.getDate() != null) { + // Special case : we have an absolute time but a relative day + if(injectTime.isRelativeDay()) { + injectTime.setDate(injectTime.getDate().plus(injectTime.getRelativeDayNumber(), ChronoUnit.DAYS)); + } + // Great, we already have an absolute date for this one + // If we are the first, good, nothing more to do + // Otherwise, we need to get the date of the first row to set the depends on + if(integerInjectTimeEntry.getKey() != firstRow) { + Instant firstDate = mapInstantByRowIndex.get(firstRow).getDate(); + injectTime.getLinkedInject().setDependsDuration(injectTime.getDate().getEpochSecond() - firstDate.getEpochSecond()); + } + } else { + // We don't have an absolute date so we need to deduce it from another row + if(injectTime.getRelativeDayNumber() < 0 + || (injectTime.getRelativeDayNumber() == 0 && injectTime.getRelativeHourNumber() < 0) + || (injectTime.getRelativeDayNumber() == 0 && injectTime.getRelativeHourNumber() == 0 && injectTime.getRelativeMinuteNumber() < 0)) { + // We are in the past, so we need to explore the future to find the next absolute date + Optional> firstFutureWithAbsolute = + mapInstantByRowIndex.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .filter(entry -> entry.getKey() > integerInjectTimeEntry.getKey() + && entry.getValue().getDate() != null) + .findFirst(); + if(firstFutureWithAbsolute.isPresent()) { + injectTime.setDate(firstFutureWithAbsolute.get().getValue() + .getDate() + .plus(injectTime.getRelativeDayNumber(), ChronoUnit.DAYS) + .plus(injectTime.getRelativeHourNumber(), ChronoUnit.HOURS) + .plus(injectTime.getRelativeMinuteNumber(), ChronoUnit.MINUTES) + ); + } else { + importMessages.add(new ImportMessage(ImportMessage.MessageLevel.ERROR, + ImportMessage.ErrorCode.DATE_SET_IN_PAST, + Map.of("row_num", String.valueOf(integerInjectTimeEntry.getKey())))); + } + } else { + // We are in the future, so we need to explore the past to find an absolute date + Optional> firstPastWithAbsolute = + mapInstantByRowIndex.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .filter(entry -> entry.getKey() < integerInjectTimeEntry.getKey() + && entry.getValue().getDate() != null) + .min(Map.Entry.comparingByKey(Comparator.reverseOrder())); + if(firstPastWithAbsolute.isPresent()) { + injectTime.setDate(firstPastWithAbsolute.get().getValue() + .getDate() + .plus(injectTime.getRelativeDayNumber(), ChronoUnit.DAYS) + .plus(injectTime.getRelativeHourNumber(), ChronoUnit.HOURS) + .plus(injectTime.getRelativeMinuteNumber(), ChronoUnit.MINUTES) + ); + } else { + importMessages.add(new ImportMessage(ImportMessage.MessageLevel.ERROR, + ImportMessage.ErrorCode.DATE_SET_IN_FUTURE, + Map.of("row_num", String.valueOf(integerInjectTimeEntry.getKey())))); + } + } + } + } + ); + + processDateToAbsolute(mapInstantByRowIndex); + } + + return importMessages; + } + + private void processDateToAbsolute(Map mapInstantByRowIndex) { + Optional earliestAbsoluteDate = mapInstantByRowIndex.values().stream() + .map(InjectTime::getDate) + .filter(Objects::nonNull) + .min(Comparator.naturalOrder()); + + // If we have an earliest date, we calculate the dates depending on the earliest one + earliestAbsoluteDate.ifPresent( + earliestInstant -> mapInstantByRowIndex.values().stream().filter(injectTime -> injectTime.getDate() != null) + .forEach(injectTime -> { + injectTime.getLinkedInject().setDependsDuration(injectTime.getDate().getEpochSecond() - earliestInstant.getEpochSecond()); + }) + ); + } + + private String getDateAsStringFromCell(Row row, String cellColumn) { + if(row.getCell(CellReference.convertColStringToIndex(cellColumn)) != null) { + Cell cell = row.getCell(CellReference.convertColStringToIndex(cellColumn)); + if(cell.getCellType() == CellType.STRING) { + return cell.getStringCellValue(); + } else if(cell.getCellType() == CellType.NUMERIC) { + return cell.getDateCellValue().toString(); + } + } + return ""; + } + + private String getValueAsString(Row row, String cellColumn) { + if(row.getCell(CellReference.convertColStringToIndex(cellColumn)) != null) { + Cell cell = row.getCell(CellReference.convertColStringToIndex(cellColumn)); + if(cell.getCellType() == CellType.STRING) { + return cell.getStringCellValue(); + } else if(cell.getCellType() == CellType.NUMERIC) { + return Double.valueOf(cell.getNumericCellValue()).toString(); + } + } + return ""; + } + + private Double getValueAsDouble(Row row, String cellColumn) { + if(row.getCell(CellReference.convertColStringToIndex(cellColumn)) != null) { + Cell cell = row.getCell(CellReference.convertColStringToIndex(cellColumn)); + if(cell.getCellType() == CellType.STRING) { + return Double.valueOf(cell.getStringCellValue()); + } else if(cell.getCellType() == CellType.NUMERIC) { + return cell.getNumericCellValue(); + } + } + return 0.0; + } + + // -- TEST -- private Map mapOfInjectsExpectations(@NotNull final List rawInjects) { return this.injectExpectationRepository @@ -262,5 +1002,68 @@ private Map mapOfRawTeams( ).distinct().toList()).stream().collect(Collectors.toMap(RawTeam::getTeam_id, Function.identity())); } + // -- CRITERIA BUILDER -- + + private void selectForInject(CriteriaBuilder cb, CriteriaQuery cq, Root injectRoot) { + // Joins + Join injectExerciseJoin = createLeftJoin(injectRoot, "exercise"); + Join injectScenarioJoin = createLeftJoin(injectRoot, "scenario"); + Join injectorContractJoin = createLeftJoin(injectRoot, "injectorContract"); + Join injectorJoin = injectorContractJoin.join("injector", JoinType.LEFT); + // Array aggregations + Expression tagIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "tags"); + Expression teamIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "teams"); + Expression assetIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "assets"); + Expression assetGroupIdsExpression = createJoinArrayAggOnId(cb, injectRoot, "assetGroups"); + + // SELECT + cq.multiselect( + injectRoot.get("id").alias("inject_id"), + injectRoot.get("title").alias("inject_title"), + injectRoot.get("enabled").alias("inject_enabled"), + injectRoot.get("content").alias("inject_content"), + injectRoot.get("allTeams").alias("inject_all_teams"), + injectExerciseJoin.get("id").alias("inject_exercise"), + injectScenarioJoin.get("id").alias("inject_scenario"), + injectRoot.get("dependsDuration").alias("inject_depends_duration"), + injectorContractJoin.alias("inject_injector_contract"), + tagIdsExpression.alias("inject_tags"), + teamIdsExpression.alias("inject_teams"), + assetIdsExpression.alias("inject_assets"), + assetGroupIdsExpression.alias("inject_asset_groups"), + injectorJoin.get("type").alias("inject_type") + ).distinct(true); + + // GROUP BY + cq.groupBy(Arrays.asList( + injectRoot.get("id"), + injectExerciseJoin.get("id"), + injectScenarioJoin.get("id"), + injectorContractJoin.get("id"), + injectorJoin.get("id") + )); + } + + private List execInject(TypedQuery query) { + return query.getResultList() + .stream() + .map(tuple -> new InjectOutput( + tuple.get("inject_id", String.class), + tuple.get("inject_title", String.class), + tuple.get("inject_enabled", Boolean.class), + tuple.get("inject_content", ObjectNode.class), + tuple.get("inject_all_teams", Boolean.class), + tuple.get("inject_exercise", String.class), + tuple.get("inject_scenario", String.class), + tuple.get("inject_depends_duration", Long.class), + tuple.get("inject_injector_contract", InjectorContract.class), + tuple.get("inject_tags", String[].class), + tuple.get("inject_teams", String[].class), + tuple.get("inject_assets", String[].class), + tuple.get("inject_asset_groups", String[].class), + tuple.get("inject_type", String.class) + )) + .toList(); + } } diff --git a/openbas-api/src/main/java/io/openbas/service/InjectTime.java b/openbas-api/src/main/java/io/openbas/service/InjectTime.java new file mode 100644 index 0000000000..55bcb1eeec --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/service/InjectTime.java @@ -0,0 +1,22 @@ +package io.openbas.service; + +import io.openbas.database.model.Inject; +import lombok.Data; + +import java.time.Instant; +import java.time.format.DateTimeFormatter; + +@Data +public class InjectTime { + private boolean isRelativeDay = false; + private boolean isRelativeHour = false; + private boolean isRelativeMinute = false; + private boolean specifyDays = true; + private DateTimeFormatter formatter = DateTimeFormatter.ISO_DATE_TIME; + private Instant date; + private String unformattedDate; + private int relativeDayNumber; + private int relativeHourNumber; + private int relativeMinuteNumber; + private Inject linkedInject; +} diff --git a/openbas-api/src/main/java/io/openbas/service/MapperService.java b/openbas-api/src/main/java/io/openbas/service/MapperService.java new file mode 100644 index 0000000000..dc5f593b1c --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/service/MapperService.java @@ -0,0 +1,179 @@ +package io.openbas.service; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.model.InjectImporter; +import io.openbas.database.model.InjectorContract; +import io.openbas.database.model.RuleAttribute; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.database.repository.InjectorContractRepository; +import io.openbas.rest.exception.ElementNotFoundException; +import io.openbas.rest.mapper.form.*; +import lombok.RequiredArgsConstructor; +import org.springframework.stereotype.Service; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +@RequiredArgsConstructor +@Service +public class MapperService { + + private final ImportMapperRepository importMapperRepository; + + private final InjectorContractRepository injectorContractRepository; + + /** + * Create and save an ImportMapper object from a MapperAddInput one + * @param importMapperAddInput The input from the call + * @return The created ImportMapper + */ + public ImportMapper createAndSaveImportMapper(ImportMapperAddInput importMapperAddInput) { + ImportMapper importMapper = createImportMapper(importMapperAddInput); + + return importMapperRepository.save(importMapper); + } + + public ImportMapper createImportMapper(ImportMapperAddInput importMapperAddInput) { + ImportMapper importMapper = new ImportMapper(); + importMapper.setUpdateAttributes(importMapperAddInput); + importMapper.setInjectImporters(new ArrayList<>()); + + Map mapInjectorContracts = getMapOfInjectorContracts( + importMapperAddInput.getImporters() + .stream() + .map(InjectImporterAddInput::getInjectorContractId) + .toList() + ); + + importMapperAddInput.getImporters().forEach( + injectImporterInput -> { + InjectImporter injectImporter = new InjectImporter(); + injectImporter.setInjectorContract(mapInjectorContracts.get(injectImporterInput.getInjectorContractId())); + injectImporter.setImportTypeValue(injectImporterInput.getInjectTypeValue()); + injectImporter.setRuleAttributes(new ArrayList<>()); + injectImporterInput.getRuleAttributes().forEach(ruleAttributeInput -> { + RuleAttribute ruleAttribute = new RuleAttribute(); + ruleAttribute.setColumns(ruleAttributeInput.getColumns()); + ruleAttribute.setName(ruleAttributeInput.getName()); + ruleAttribute.setDefaultValue(ruleAttributeInput.getDefaultValue()); + ruleAttribute.setAdditionalConfig(ruleAttributeInput.getAdditionalConfig()); + injectImporter.getRuleAttributes().add(ruleAttribute); + }); + importMapper.getInjectImporters().add(injectImporter); + } + ); + + return importMapper; + } + + /** + * Update an ImportMapper object from a MapperUpdateInput one + * @param mapperId the id of the mapper that needs to be updated + * @param importMapperUpdateInput The input from the call + * @return The updated ImportMapper + */ + public ImportMapper updateImportMapper(String mapperId, ImportMapperUpdateInput importMapperUpdateInput) { + ImportMapper importMapper = importMapperRepository.findById(UUID.fromString(mapperId)).orElseThrow(ElementNotFoundException::new); + importMapper.setUpdateAttributes(importMapperUpdateInput); + importMapper.setUpdateDate(Instant.now()); + + Map mapInjectorContracts = getMapOfInjectorContracts( + importMapperUpdateInput.getImporters() + .stream() + .map(InjectImporterUpdateInput::getInjectorContractId) + .toList() + ); + + updateInjectImporter(importMapperUpdateInput.getImporters(), importMapper.getInjectImporters(), mapInjectorContracts); + + return importMapperRepository.save(importMapper); + } + + /** + * Gets a map of injector contracts by ids + * @param ids The ids of the injector contracts we want + * @return The map of injector contracts by ids + */ + private Map getMapOfInjectorContracts(List ids) { + return StreamSupport.stream(injectorContractRepository.findAllById(ids).spliterator(), false) + .collect(Collectors.toMap(InjectorContract::getId, Function.identity())); + } + + /** + * Updates rule attributes from a list of input + * @param ruleAttributesInput the list of rule attributes input + * @param ruleAttributes the list of rule attributes to update + */ + private void updateRuleAttributes(List ruleAttributesInput, List ruleAttributes) { + // First, we remove the entities that are no longer linked to the mapper + ruleAttributes.removeIf(ruleAttribute -> ruleAttributesInput.stream().noneMatch(importerInput -> ruleAttribute.getId().equals(importerInput.getId()))); + + // Then we update the existing ones + ruleAttributes.forEach(ruleAttribute -> { + RuleAttributeUpdateInput ruleAttributeInput = ruleAttributesInput.stream() + .filter(ruleAttributeUpdateInput -> ruleAttribute.getId().equals(ruleAttributeUpdateInput.getId())) + .findFirst() + .orElseThrow(ElementNotFoundException::new); + ruleAttribute.setUpdateAttributes(ruleAttributeInput); + }); + + // Then we add the new ones + ruleAttributesInput.forEach(ruleAttributeUpdateInput -> { + if (ruleAttributeUpdateInput.getId() == null || ruleAttributeUpdateInput.getId().isBlank()) { + RuleAttribute ruleAttribute = new RuleAttribute(); + ruleAttribute.setColumns(ruleAttributeUpdateInput.getColumns()); + ruleAttribute.setName(ruleAttributeUpdateInput.getName()); + ruleAttribute.setDefaultValue(ruleAttributeUpdateInput.getDefaultValue()); + ruleAttribute.setAdditionalConfig(ruleAttributeUpdateInput.getAdditionalConfig()); + ruleAttributes.add(ruleAttribute); + } + }); + } + + /** + * Updates a list of inject importers from an input one + * @param injectImportersInput the input + * @param injectImporters the inject importers to update + * @param mapInjectorContracts a map of injector contracts by contract id + */ + private void updateInjectImporter(List injectImportersInput, List injectImporters, Map mapInjectorContracts) { + // First, we remove the entities that are no longer linked to the mapper + injectImporters.removeIf(importer -> !injectImportersInput.stream().anyMatch(importerInput -> importer.getId().equals(importerInput.getId()))); + + // Then we update the existing ones + injectImporters.forEach(injectImporter -> { + InjectImporterUpdateInput injectImporterInput = injectImportersInput.stream() + .filter(injectImporterUpdateInput -> injectImporter.getId().equals(injectImporterUpdateInput.getId())) + .findFirst() + .orElseThrow(ElementNotFoundException::new); + injectImporter.setUpdateAttributes(injectImporterInput); + updateRuleAttributes(injectImporterInput.getRuleAttributes(), injectImporter.getRuleAttributes()); + }); + + // Then we add the new ones + injectImportersInput.forEach(injectImporterUpdateInput -> { + if (injectImporterUpdateInput.getId() == null || injectImporterUpdateInput.getId().isBlank()) { + InjectImporter injectImporter = new InjectImporter(); + injectImporter.setInjectorContract(mapInjectorContracts.get(injectImporterUpdateInput.getInjectorContractId())); + injectImporter.setImportTypeValue(injectImporterUpdateInput.getInjectTypeValue()); + injectImporter.setRuleAttributes(new ArrayList<>()); + injectImporterUpdateInput.getRuleAttributes().forEach(ruleAttributeInput -> { + RuleAttribute ruleAttribute = new RuleAttribute(); + ruleAttribute.setColumns(ruleAttributeInput.getColumns()); + ruleAttribute.setName(ruleAttributeInput.getName()); + ruleAttribute.setDefaultValue(ruleAttributeInput.getDefaultValue()); + ruleAttribute.setAdditionalConfig(ruleAttributeInput.getAdditionalConfig()); + injectImporter.getRuleAttributes().add(ruleAttribute); + }); + injectImporters.add(injectImporter); + } + }); + } + +} diff --git a/openbas-api/src/main/java/io/openbas/utils/InjectUtils.java b/openbas-api/src/main/java/io/openbas/utils/InjectUtils.java new file mode 100644 index 0000000000..75a7bb0e0a --- /dev/null +++ b/openbas-api/src/main/java/io/openbas/utils/InjectUtils.java @@ -0,0 +1,25 @@ +package io.openbas.utils; + +import org.apache.commons.lang3.StringUtils; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.CellType; +import org.apache.poi.ss.usermodel.Row; + +public class InjectUtils { + + public static boolean checkIfRowIsEmpty(Row row) { + if (row == null) { + return true; + } + if (row.getLastCellNum() <= 0) { + return true; + } + for (int cellNum = row.getFirstCellNum(); cellNum < row.getLastCellNum(); cellNum++) { + Cell cell = row.getCell(cellNum); + if (cell != null && cell.getCellType() != CellType.BLANK && StringUtils.isNotBlank(cell.toString())) { + return false; + } + } + return true; + } +} diff --git a/openbas-api/src/test/java/io/openbas/rest/MapperApiTest.java b/openbas-api/src/test/java/io/openbas/rest/MapperApiTest.java new file mode 100644 index 0000000000..07ec941a93 --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/rest/MapperApiTest.java @@ -0,0 +1,248 @@ +package io.openbas.rest; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.jayway.jsonpath.JsonPath; +import io.openbas.database.model.ImportMapper; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.rest.mapper.MapperApi; +import io.openbas.rest.mapper.form.ImportMapperAddInput; +import io.openbas.rest.mapper.form.ImportMapperUpdateInput; +import io.openbas.rest.scenario.form.InjectsImportTestInput; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.service.InjectService; +import io.openbas.service.MapperService; +import io.openbas.utils.fixtures.PaginationFixture; +import io.openbas.utils.mockMapper.MockMapperUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Pageable; +import org.springframework.http.MediaType; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; +import org.springframework.util.ResourceUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; +import java.lang.reflect.Field; +import java.util.List; +import java.util.Optional; +import java.util.UUID; + +import static io.openbas.utils.JsonUtils.asJsonString; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest +@ExtendWith(MockitoExtension.class) +public class MapperApiTest { + + private MockMvc mvc; + + @Mock + private ImportMapperRepository importMapperRepository; + + @Mock + private MapperService mapperService; + @Mock + private InjectService injectService; + + private MapperApi mapperApi; + + @Autowired + private ObjectMapper objectMapper; + + @BeforeEach + void before() throws IllegalAccessException, NoSuchFieldException { + // Injecting mocks into the controller + mapperApi = new MapperApi(importMapperRepository, mapperService, injectService); + + Field sessionContextField = MapperApi.class.getSuperclass().getDeclaredField("mapper"); + sessionContextField.setAccessible(true); + sessionContextField.set(mapperApi, objectMapper); + + mvc = MockMvcBuilders.standaloneSetup(mapperApi) + .build(); + } + + // -- SCENARIOS -- + + @DisplayName("Test search of mappers") + @Test + void searchMappers() throws Exception { + // -- PREPARE -- + List importMappers = List.of(MockMapperUtils.createImportMapper()); + Pageable pageable = PageRequest.of(0, 10); + PageImpl page = new PageImpl<>(importMappers, pageable, importMappers.size()); + when(importMapperRepository.findAll(any(), any())).thenReturn(page); + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.post("/api/mappers/search") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(PaginationFixture.getDefault().textSearch("").build()))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(JsonPath.read(response, "$.content[0].import_mapper_id"), importMappers.get(0).getId()); + } + + @DisplayName("Test search of a specific mapper") + @Test + void searchSpecificMapper() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + when(importMapperRepository.findById(any())).thenReturn(Optional.of(importMapper)); + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.get("/api/mappers/" + importMapper.getId()) + .contentType(MediaType.APPLICATION_JSON)) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(JsonPath.read(response, "$.import_mapper_id"), importMapper.getId()); + } + + @DisplayName("Test create a mapper") + @Test + void createMapper() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperAddInput importMapperInput = new ImportMapperAddInput(); + importMapperInput.setName("Test"); + importMapperInput.setInjectTypeColumn("B"); + when(mapperService.createAndSaveImportMapper(any())).thenReturn(importMapper); + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.post("/api/mappers/") + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(importMapperInput))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(JsonPath.read(response, "$.import_mapper_id"), importMapper.getId()); + } + + @DisplayName("Test delete a specific mapper") + @Test + void deleteSpecificMapper() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + // -- EXECUTE -- + this.mvc + .perform(MockMvcRequestBuilders.delete("/api/mappers/" + importMapper.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(PaginationFixture.getDefault().textSearch("").build()))) + .andExpect(status().is2xxSuccessful()); + + verify(importMapperRepository, times(1)).deleteById(any()); + } + + @DisplayName("Test update a specific mapper by using new rule attributes and new inject importer") + @Test + void updateSpecificMapper() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperUpdateInput importMapperInput = new ImportMapperUpdateInput(); + importMapperInput.setName("New name"); + importMapperInput.setInjectTypeColumn("B"); + when(mapperService.updateImportMapper(any(), any())).thenReturn(importMapper); + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.put("/api/mappers/" + importMapper.getId()) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(importMapperInput))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(JsonPath.read(response, "$.import_mapper_id"), importMapper.getId()); + } + + + + @DisplayName("Test store xls") + @Test + void testStoreXls() throws Exception { + // -- PREPARE -- + // Getting a test file + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_1.xlsx"); + + InputStream in = new FileInputStream(testFile); + MockMultipartFile xlsFile = new MockMultipartFile("file", + "my-awesome-file.xls", + "application/xlsx", + in.readAllBytes()); + + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.multipart("/api/mappers/store") + .file(xlsFile)) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + } + + + + @DisplayName("Test testing an import xls") + @Test + void testTestingXls() throws Exception { + // -- PREPARE -- + InjectsImportTestInput injectsImportInput = new InjectsImportTestInput(); + injectsImportInput.setImportMapper(new ImportMapperAddInput()); + injectsImportInput.setName("TEST"); + injectsImportInput.setTimezoneOffset(120); + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + + injectsImportInput.getImportMapper().setName("TEST"); + + when(injectService.importInjectIntoScenarioFromXLS(any(), any(), any(), any(), anyInt(), anyBoolean())).thenReturn(new ImportTestSummary()); + when(mapperService.createImportMapper(any())).thenReturn(importMapper); + + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.post("/api/mappers/store/{importId}", UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(injectsImportInput))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + } + +} diff --git a/openbas-api/src/test/java/io/openbas/scenario/ScenarioImportApiTest.java b/openbas-api/src/test/java/io/openbas/scenario/ScenarioImportApiTest.java new file mode 100644 index 0000000000..a90ab5c21b --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/scenario/ScenarioImportApiTest.java @@ -0,0 +1,113 @@ +package io.openbas.scenario; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.rest.scenario.ScenarioImportApi; +import io.openbas.rest.scenario.form.InjectsImportInput; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.service.InjectService; +import io.openbas.service.ScenarioService; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +import java.util.Optional; +import java.util.UUID; + +import static io.openbas.utils.JsonUtils.asJsonString; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +@SpringBootTest +@ExtendWith(MockitoExtension.class) +public class ScenarioImportApiTest { + + private MockMvc mvc; + + @Mock + private InjectService injectService; + + @Mock + private ImportMapperRepository importMapperRepository; + + @Mock + private ScenarioService scenarioService; + + private ScenarioImportApi scenarioImportApi; + + private String SCENARIO_ID; + + @BeforeEach + public void setUp() { + // Injecting mocks into the controller + scenarioImportApi = new ScenarioImportApi(injectService, importMapperRepository, scenarioService); + + SCENARIO_ID = UUID.randomUUID().toString(); + + mvc = MockMvcBuilders.standaloneSetup(scenarioImportApi) + .build(); + } + + @DisplayName("Test dry run import xls") + @Test + void testDryRunXls() throws Exception { + // -- PREPARE -- + InjectsImportInput injectsImportInput = new InjectsImportInput(); + injectsImportInput.setImportMapperId(UUID.randomUUID().toString()); + injectsImportInput.setName("TEST"); + injectsImportInput.setTimezoneOffset(120); + + when(importMapperRepository.findById(any())).thenReturn(Optional.of(new ImportMapper())); + when(injectService.importInjectIntoScenarioFromXLS(any(), any(), any(), any(), anyInt(), anyBoolean())).thenReturn(new ImportTestSummary()); + + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.post("/api/scenarios/{scenarioId}/xls/{importId}/dry", SCENARIO_ID, UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(injectsImportInput))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + } + + @DisplayName("Test import xls") + @Test + void testImportXls() throws Exception { + // -- PREPARE -- + InjectsImportInput injectsImportInput = new InjectsImportInput(); + injectsImportInput.setImportMapperId(UUID.randomUUID().toString()); + injectsImportInput.setName("TEST"); + injectsImportInput.setTimezoneOffset(120); + + when(importMapperRepository.findById(any())).thenReturn(Optional.of(new ImportMapper())); + when(injectService.importInjectIntoScenarioFromXLS(any(), any(), any(), any(), anyInt(), anyBoolean())).thenReturn(new ImportTestSummary()); + + // -- EXECUTE -- + String response = this.mvc + .perform(MockMvcRequestBuilders.post("/api/scenarios/{scenarioId}/xls/{importId}/import", SCENARIO_ID, UUID.randomUUID().toString()) + .contentType(MediaType.APPLICATION_JSON) + .content(asJsonString(injectsImportInput))) + .andExpect(status().is2xxSuccessful()) + .andReturn() + .getResponse() + .getContentAsString(); + + // -- ASSERT -- + assertNotNull(response); + } + +} diff --git a/openbas-api/src/test/java/io/openbas/service/InjectServiceTest.java b/openbas-api/src/test/java/io/openbas/service/InjectServiceTest.java new file mode 100644 index 0000000000..43b74cc996 --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/service/InjectServiceTest.java @@ -0,0 +1,671 @@ +package io.openbas.service; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.openbas.config.OpenBASOAuth2User; +import io.openbas.config.SessionHelper; +import io.openbas.database.model.*; +import io.openbas.database.repository.*; +import io.openbas.rest.exception.BadRequestException; +import io.openbas.rest.scenario.form.InjectsImportInput; +import io.openbas.rest.scenario.response.ImportMessage; +import io.openbas.rest.scenario.response.ImportPostSummary; +import io.openbas.rest.scenario.response.ImportTestSummary; +import io.openbas.utils.CustomMockMultipartFile; +import jakarta.annotation.Resource; +import org.apache.commons.io.FilenameUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.util.ResourceUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.time.Month; +import java.time.ZoneOffset; +import java.util.*; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +@ExtendWith(MockitoExtension.class) +class InjectServiceTest { + + @Mock + InjectRepository injectRepository; + @Mock + InjectDocumentRepository injectDocumentRepository; + @Mock + InjectExpectationRepository injectExpectationRepository; + @Mock + AssetRepository assetRepository; + @Mock + AssetGroupRepository assetGroupRepository; + @Mock + TeamRepository teamRepository; + @Mock + ScenarioTeamUserRepository scenarioTeamUserRepository; + @Mock + UserRepository userRepository; + @Mock + ScenarioService scenarioService; + @Mock + ImportMapperRepository importMapperRepository; + @InjectMocks + private InjectService injectService; + + private Scenario mockedScenario; + + private ImportMapper mockedImportMapper; + + private InjectsImportInput mockedInjectsImportInput; + @Resource + protected ObjectMapper mapper; + + @BeforeEach + void setUp() { + injectService = new InjectService(injectRepository, injectDocumentRepository, injectExpectationRepository, + assetRepository, assetGroupRepository, scenarioTeamUserRepository, teamRepository, userRepository, scenarioService); + + mockedScenario = new Scenario(); + mapper = new ObjectMapper(); + } + + @DisplayName("Post and store an XLS file") + @Test + void postAnXLSFile() throws Exception { + // -- PREPARE -- + // Getting a test file + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_1.xlsx"); + + InputStream in = new FileInputStream(testFile); + MockMultipartFile xlsFile = new MockMultipartFile("file", + "my-awesome-file.xls", + "application/xlsx", + in.readAllBytes()); + + ImportPostSummary response = injectService.storeXlsFileForImport(xlsFile); + + // -- ASSERT -- + assertNotNull(response); + try { + UUID.fromString(response.getImportId()); + } catch (Exception ex) { + fail(); + } + assertEquals(1, response.getAvailableSheets().size()); + assertEquals("CHECKLIST", response.getAvailableSheets().get(0)); + } + + @DisplayName("Post and store a corrupted XLS file") + @Test + void postACorruptedXLSFile() throws Exception { + // Getting a test file + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_1.xlsx"); + // -- PREPARE -- + InputStream in = new FileInputStream(testFile); + MockMultipartFile xlsFile = new CustomMockMultipartFile("file", + "my-awesome-file.xls", + "application/xlsx", + in.readAllBytes()); + + // -- EXECUTE -- + try { + injectService.storeXlsFileForImport(xlsFile); + fail(); + } catch(Exception ex) { + assertTrue(ex instanceof BadRequestException); + } + } + + @DisplayName("Import an XLS file with relative date") + @Test + void testImportXlsRelativeDate() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_1.xlsx"); + createTempFile(testFile, fileID); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + mockedScenario.setId(UUID.randomUUID().toString()); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + verify(teamRepository, times(1)).save(any()); + assertEquals(30 * 24 * 60 * 60, importTestSummary.getInjects().getLast().getDependsDuration()); + + ObjectNode jsonNodeMail = (ObjectNode) mapper.readTree("{\"message\":\"message1\",\"expectations\":[{\"expectation_description\":\"expectation\",\"expectation_name\":\"expectation done\",\"expectation_score\":100,\"expectation_type\":\"MANUAL\",\"expectation_expectation_group\":false}]}"); + assertEquals(jsonNodeMail, importTestSummary.getInjects().getFirst().getContent()); + + ObjectNode jsonNodeSms = (ObjectNode) mapper.readTree("{\"subject\":\"subject\",\"body\":\"message2\",\"expectations\":[{\"expectation_description\":\"expectation\",\"expectation_name\":\"expectation done\",\"expectation_score\":100,\"expectation_type\":\"MANUAL\",\"expectation_expectation_group\":false}]}"); + assertEquals(jsonNodeSms, importTestSummary.getInjects().getLast().getContent()); + } + } + + @DisplayName("Import a non existing XLS file") + @Test + void testImportXlsBadFile() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + String fileID = UUID.randomUUID().toString(); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + try { + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + fail(); + } catch (Exception ex) { + assertTrue(ex instanceof BadRequestException); + } + } + } + + @DisplayName("Import an XLS file and have several matches of importer") + @Test + void testImportXlsSeveralMatches() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_1.xlsx"); + createTempFile(testFile, fileID); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + lenient().when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + + InjectImporter injectImporterMailCopy = new InjectImporter(); + injectImporterMailCopy.setId(UUID.randomUUID().toString()); + injectImporterMailCopy.setImportTypeValue(".*mail"); + injectImporterMailCopy.setRuleAttributes(new ArrayList<>()); + injectImporterMailCopy.setInjectorContract(createMailInjectorContract()); + + injectImporterMailCopy.getRuleAttributes().addAll(createRuleAttributeMail()); + mockedImportMapper.getInjectImporters().add(injectImporterMailCopy); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + assertTrue( + importTestSummary.getImportMessage().stream().anyMatch( + importMessage -> importMessage.getMessageLevel().equals(ImportMessage.MessageLevel.WARN) + && importMessage.getErrorCode().equals(ImportMessage.ErrorCode.SEVERAL_MATCHES) + ) + ); + } + } + + @DisplayName("Import an XLS file with absolute date") + @Test + void testImportXlsAbsoluteDate() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_2.xlsx"); + createTempFile(testFile, fileID); + + mockedScenario = new Scenario(); + mockedScenario.setId(UUID.randomUUID().toString()); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + mockedImportMapper.getInjectImporters().forEach(injectImporter -> { + injectImporter.setRuleAttributes(injectImporter.getRuleAttributes().stream() + .map(ruleAttribute -> { + if("trigger_time".equals(ruleAttribute.getName())) { + ruleAttribute.setAdditionalConfig(Map.of("timePattern", "dd/MM/yyyy HH'h'mm")); + } + return ruleAttribute; + }).toList() + ); + }); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + assertTrue(LocalDateTime.of(2024, Month.JUNE, 26, 0, 0) + .toInstant(ZoneOffset.of("Z")) + .equals(mockedScenario.getRecurrenceStart())); + assertTrue("0 0 7 * * *".equals(mockedScenario.getRecurrence())); + } + } + + @DisplayName("Import an XLS file with relative and absolute dates") + @Test + void testImportXlsAbsoluteAndRelativeDates() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_3.xlsx"); + createTempFile(testFile, fileID); + + mockedScenario = new Scenario(); + mockedScenario.setId(UUID.randomUUID().toString()); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + mockedImportMapper.getInjectImporters().forEach(injectImporter -> { + injectImporter.setRuleAttributes(injectImporter.getRuleAttributes().stream() + .map(ruleAttribute -> { + if("trigger_time".equals(ruleAttribute.getName())) { + ruleAttribute.setAdditionalConfig(Map.of("timePattern", "dd/MM/yyyy HH'h'mm")); + } + return ruleAttribute; + }).toList() + ); + }); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + List sortedInjects = importTestSummary.getInjects().stream() + .sorted(Comparator.comparing(Inject::getDependsDuration)) + .toList(); + + assertEquals(24 * 60 * 60, sortedInjects.get(1).getDependsDuration()); + assertEquals(24 * 60 * 60 + 5 * 60, sortedInjects.get(2).getDependsDuration()); + assertEquals(2 * 24 * 60 * 60, sortedInjects.get(3).getDependsDuration()); + } + } + + @DisplayName("Import an XLS file with relative dates and absolute hours") + @Test + void testImportXlsRelativeDatesAndAbsoluteHour() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_4.xlsx"); + createTempFile(testFile, fileID); + + mockedScenario.setId(UUID.randomUUID().toString()); + mockedScenario.setRecurrenceStart(LocalDateTime.of(2024, Month.JUNE, 26, 0, 0) + .toInstant(ZoneOffset.of("Z"))); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + mockedImportMapper.getInjectImporters().forEach(injectImporter -> { + injectImporter.setRuleAttributes(injectImporter.getRuleAttributes().stream() + .map(ruleAttribute -> { + if("trigger_time".equals(ruleAttribute.getName())) { + ruleAttribute.setAdditionalConfig(Map.of("timePattern", "HH'h'mm")); + } + return ruleAttribute; + }).toList() + ); + }); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + List sortedInjects = importTestSummary.getInjects().stream() + .sorted(Comparator.comparing(Inject::getDependsDuration)) + .toList(); + + assertEquals(24 * 60 * 60, sortedInjects.get(1).getDependsDuration()); + assertEquals(2 * 24 * 60 * 60, sortedInjects.get(2).getDependsDuration()); + assertEquals(4 * 24 * 60 * 60 + 5 * 60, sortedInjects.get(3).getDependsDuration()); + } + } + + @DisplayName("Critical message when import an XLS file with relative dates " + + "and absolute hours but no date in scenario") + @Test + void testImportXlsRelativeDatesAndAbsoluteHourCriticalMessage() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_4.xlsx"); + createTempFile(testFile, fileID); + + mockedScenario = new Scenario(); + mockedScenario.setId(UUID.randomUUID().toString()); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + mockedImportMapper.getInjectImporters().forEach(injectImporter -> { + injectImporter.setRuleAttributes(injectImporter.getRuleAttributes().stream() + .map(ruleAttribute -> { + if("trigger_time".equals(ruleAttribute.getName())) { + ruleAttribute.setAdditionalConfig(Map.of("timePattern", "HH'h'mm")); + } + return ruleAttribute; + }).toList() + ); + }); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + assertTrue(importTestSummary.getImportMessage().stream().anyMatch( + importMessage -> ImportMessage.MessageLevel.CRITICAL.equals(importMessage.getMessageLevel()) + && ImportMessage.ErrorCode.ABSOLUTE_TIME_WITHOUT_START_DATE.equals(importMessage.getErrorCode()) + )); + } + } + + @DisplayName("Import an XLS file with relative dates, hours and minutes") + @Test + void testImportXlsRelativeDatesHoursAndMinutes() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_5.xlsx"); + createTempFile(testFile, fileID); + mockedInjectsImportInput = new InjectsImportInput(); + mockedInjectsImportInput.setImportMapperId(fileID); + mockedInjectsImportInput.setName("CHECKLIST"); + mockedInjectsImportInput.setTimezoneOffset(120); + + mockedScenario = new Scenario(); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + List sortedInjects = importTestSummary.getInjects().stream() + .sorted(Comparator.comparing(Inject::getDependsDuration)) + .toList(); + + assertEquals(24 * 60 * 60, sortedInjects.get(1).getDependsDuration()); + assertEquals(((((2 * 24) + 2) * 60) - 5) * 60, sortedInjects.get(2).getDependsDuration()); + assertEquals(4 * 24 * 60 * 60, sortedInjects.get(3).getDependsDuration()); + } + } + + @DisplayName("Import an XLS file with default values") + @Test + void testImportXlsDefaultValue() throws IOException { + try (MockedStatic sessionHelper = Mockito.mockStatic(SessionHelper.class)) { + User mockedUser = new User(); + String fileID = UUID.randomUUID().toString(); + File testFile = ResourceUtils.getFile("classpath:xls-test-files/test_file_5.xlsx"); + createTempFile(testFile, fileID); + + mockedScenario = new Scenario(); + mockedScenario.setId(UUID.randomUUID().toString()); + + mockedImportMapper = createImportMapper(UUID.randomUUID().toString()); + mockedImportMapper.getInjectImporters().forEach(injectImporter -> { + injectImporter.setRuleAttributes(injectImporter.getRuleAttributes().stream() + .map(ruleAttribute -> { + if("title".equals(ruleAttribute.getName()) + || "trigger_time".equals(ruleAttribute.getName())) { + ruleAttribute.setColumns("A"); + } + return ruleAttribute; + }).toList() + ); + }); + when(userRepository.findById(any())).thenReturn(Optional.of(mockedUser)); + Team team1 = new Team(); + team1.setName("team1"); + Team team2 = new Team(); + team2.setName("team2"); + when(teamRepository.findAll()).thenReturn(List.of(team1)); + when(teamRepository.save(any())).thenReturn(team2); + + sessionHelper.when(SessionHelper::currentUser).thenReturn(new OpenBASOAuth2User(mockedUser)); + ImportTestSummary importTestSummary = + injectService.importInjectIntoScenarioFromXLS(mockedScenario, + mockedImportMapper, fileID, "CHECKLIST", 120, true); + + assertSame("title", importTestSummary.getInjects().getFirst().getTitle()); + } + } + + private void createTempFile(File testFile, String fileID) throws IOException { + InputStream in = new FileInputStream(testFile); + MockMultipartFile file = new MockMultipartFile("file", + "my-awesome-file.xls", + "application/xlsx", + in.readAllBytes()); + + // Writing the file in a temp dir + Path tempDir = Files.createDirectory(Path.of(System.getProperty("java.io.tmpdir"), fileID)); + Path tempFile = Files.createTempFile(tempDir, null, "." + FilenameUtils.getExtension(file.getOriginalFilename())); + Files.write(tempFile, file.getBytes()); + + // We're making sure the files are deleted when the test stops + tempDir.toFile().deleteOnExit(); + tempFile.toFile().deleteOnExit(); + } + + private ImportMapper createImportMapper(String id) throws JsonProcessingException { + ImportMapper importMapper = new ImportMapper(); + importMapper.setName("test import mapper"); + importMapper.setId(id); + importMapper.setInjectTypeColumn("B"); + importMapper.setInjectImporters(new ArrayList<>()); + + InjectImporter injectImporterSms = new InjectImporter(); + injectImporterSms.setId(UUID.randomUUID().toString()); + injectImporterSms.setImportTypeValue(".*(sms|SMS).*"); + injectImporterSms.setRuleAttributes(new ArrayList<>()); + injectImporterSms.setInjectorContract(createSmsInjectorContract()); + + injectImporterSms.getRuleAttributes().addAll(createRuleAttributeSms()); + + InjectImporter injectImporterMail = new InjectImporter(); + injectImporterMail.setId(UUID.randomUUID().toString()); + injectImporterMail.setImportTypeValue(".*mail.*"); + injectImporterMail.setRuleAttributes(new ArrayList<>()); + injectImporterMail.setInjectorContract(createMailInjectorContract()); + + injectImporterMail.getRuleAttributes().addAll(createRuleAttributeMail()); + + importMapper.getInjectImporters().add(injectImporterSms); + importMapper.getInjectImporters().add(injectImporterMail); + + return importMapper; + } + + private InjectorContract createSmsInjectorContract() throws JsonProcessingException { + InjectorContract injectorContract = new InjectorContract(); + ObjectNode jsonNode = (ObjectNode) mapper.readTree("{\"config\":{\"type\":\"openbas_ovh_sms\",\"expose\":true,\"label\":{\"en\":\"SMS (OVH)\"},\"color_dark\":\"#9c27b0\",\"color_light\":\"#9c27b0\"},\"label\":{\"en\":\"Send a SMS\",\"fr\":\"Envoyer un SMS\"},\"manual\":false,\"fields\":[{\"key\":\"teams\",\"label\":\"Teams\",\"mandatory\":true,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"cardinality\":\"n\",\"defaultValue\":[],\"type\":\"team\"},{\"key\":\"message\",\"label\":\"Message\",\"mandatory\":true,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"defaultValue\":\"\",\"richText\":false,\"type\":\"textarea\"},{\"key\":\"expectations\",\"label\":\"Expectations\",\"mandatory\":false,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"cardinality\":\"n\",\"defaultValue\":[],\"predefinedExpectations\":[],\"type\":\"expectation\"}],\"variables\":[{\"key\":\"user\",\"label\":\"User that will receive the injection\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[{\"key\":\"user.id\",\"label\":\"Id of the user in the platform\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.email\",\"label\":\"Email of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.firstname\",\"label\":\"Firstname of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.lastname\",\"label\":\"Lastname of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.lang\",\"label\":\"Lang of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}]},{\"key\":\"exercise\",\"label\":\"Exercise of the current injection\",\"type\":\"Object\",\"cardinality\":\"1\",\"children\":[{\"key\":\"exercise.id\",\"label\":\"Id of the user in the platform\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"exercise.name\",\"label\":\"Name of the exercise\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"exercise.description\",\"label\":\"Description of the exercise\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}]},{\"key\":\"teams\",\"label\":\"List of team name for the injection\",\"type\":\"String\",\"cardinality\":\"n\",\"children\":[]},{\"key\":\"player_uri\",\"label\":\"Player interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"challenges_uri\",\"label\":\"Challenges interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"scoreboard_uri\",\"label\":\"Scoreboard interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"lessons_uri\",\"label\":\"Lessons learned interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}],\"context\":{},\"contract_id\":\"e9e902bc-b03d-4223-89e1-fca093ac79dd\",\"contract_attack_patterns_external_ids\":[],\"is_atomic_testing\":true,\"needs_executor\":false,\"platforms\":[\"Service\"]}"); + injectorContract.setConvertedContent(jsonNode); + + return injectorContract; + } + + private InjectorContract createMailInjectorContract() throws JsonProcessingException { + InjectorContract injectorContract = new InjectorContract(); + ObjectNode jsonNode = (ObjectNode) mapper.readTree("{\"config\":{\"type\":\"openbas_email\",\"expose\":true,\"label\":{\"en\":\"Email\",\"fr\":\"Email\"},\"color_dark\":\"#cddc39\",\"color_light\":\"#cddc39\"},\"label\":{\"en\":\"Send individual mails\",\"fr\":\"Envoyer des mails individuels\"},\"manual\":false,\"fields\":[{\"key\":\"teams\",\"label\":\"Teams\",\"mandatory\":true,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"cardinality\":\"n\",\"defaultValue\":[],\"type\":\"team\"},{\"key\":\"subject\",\"label\":\"Subject\",\"mandatory\":true,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"defaultValue\":\"\",\"type\":\"text\"},{\"key\":\"body\",\"label\":\"Body\",\"mandatory\":true,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"defaultValue\":\"\",\"richText\":true,\"type\":\"textarea\"},{\"key\":\"encrypted\",\"label\":\"Encrypted\",\"mandatory\":false,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"defaultValue\":false,\"type\":\"checkbox\"},{\"key\":\"attachments\",\"label\":\"Attachments\",\"mandatory\":false,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"cardinality\":\"n\",\"defaultValue\":[],\"type\":\"attachment\"},{\"key\":\"expectations\",\"label\":\"Expectations\",\"mandatory\":false,\"readOnly\":false,\"mandatoryGroups\":null,\"linkedFields\":[],\"linkedValues\":[],\"cardinality\":\"n\",\"defaultValue\":[],\"predefinedExpectations\":[],\"type\":\"expectation\"}],\"variables\":[{\"key\":\"document_uri\",\"label\":\"Http user link to upload the document (only for document expectation)\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user\",\"label\":\"User that will receive the injection\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[{\"key\":\"user.id\",\"label\":\"Id of the user in the platform\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.email\",\"label\":\"Email of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.firstname\",\"label\":\"Firstname of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.lastname\",\"label\":\"Lastname of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"user.lang\",\"label\":\"Lang of the user\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}]},{\"key\":\"exercise\",\"label\":\"Exercise of the current injection\",\"type\":\"Object\",\"cardinality\":\"1\",\"children\":[{\"key\":\"exercise.id\",\"label\":\"Id of the user in the platform\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"exercise.name\",\"label\":\"Name of the exercise\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"exercise.description\",\"label\":\"Description of the exercise\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}]},{\"key\":\"teams\",\"label\":\"List of team name for the injection\",\"type\":\"String\",\"cardinality\":\"n\",\"children\":[]},{\"key\":\"player_uri\",\"label\":\"Player interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"challenges_uri\",\"label\":\"Challenges interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"scoreboard_uri\",\"label\":\"Scoreboard interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]},{\"key\":\"lessons_uri\",\"label\":\"Lessons learned interface platform link\",\"type\":\"String\",\"cardinality\":\"1\",\"children\":[]}],\"context\":{},\"contract_id\":\"138ad8f8-32f8-4a22-8114-aaa12322bd09\",\"contract_attack_patterns_external_ids\":[],\"is_atomic_testing\":true,\"needs_executor\":false,\"platforms\":[\"Service\"]}"); + injectorContract.setConvertedContent(jsonNode); + + return injectorContract; + } + + private List createRuleAttributeSms() { + List results = new ArrayList<>(); + RuleAttribute ruleAttributeTitle = new RuleAttribute(); + ruleAttributeTitle.setName("title"); + ruleAttributeTitle.setColumns("B"); + ruleAttributeTitle.setDefaultValue("title"); + + RuleAttribute ruleAttributeDescription = new RuleAttribute(); + ruleAttributeDescription.setName("description"); + ruleAttributeDescription.setColumns("G"); + ruleAttributeDescription.setDefaultValue("description"); + + RuleAttribute ruleAttributeTriggerTime = new RuleAttribute(); + ruleAttributeTriggerTime.setName("trigger_time"); + ruleAttributeTriggerTime.setColumns("C"); + ruleAttributeTriggerTime.setDefaultValue("trigger_time"); + ruleAttributeTriggerTime.setAdditionalConfig(Map.of("timePattern", "")); + + RuleAttribute ruleAttributeMessage = new RuleAttribute(); + ruleAttributeMessage.setName("message"); + ruleAttributeMessage.setColumns("F"); + ruleAttributeMessage.setDefaultValue("message"); + + RuleAttribute ruleAttributeTeams = new RuleAttribute(); + ruleAttributeTeams.setName("teams"); + ruleAttributeTeams.setColumns("D"); + ruleAttributeTeams.setDefaultValue("teams"); + + RuleAttribute ruleAttributeExpectationScore = new RuleAttribute(); + ruleAttributeExpectationScore.setName("expectation_score"); + ruleAttributeExpectationScore.setColumns("J"); + ruleAttributeExpectationScore.setDefaultValue("500"); + + RuleAttribute ruleAttributeExpectationName = new RuleAttribute(); + ruleAttributeExpectationName.setName("expectation_name"); + ruleAttributeExpectationName.setColumns("I"); + ruleAttributeExpectationName.setDefaultValue("name"); + + RuleAttribute ruleAttributeExpectationDescription = new RuleAttribute(); + ruleAttributeExpectationDescription.setName("expectation_description"); + ruleAttributeExpectationDescription.setColumns("H"); + ruleAttributeExpectationDescription.setDefaultValue("description"); + + results.add(ruleAttributeTitle); + results.add(ruleAttributeDescription); + results.add(ruleAttributeTriggerTime); + results.add(ruleAttributeMessage); + results.add(ruleAttributeTeams); + results.add(ruleAttributeExpectationScore); + results.add(ruleAttributeExpectationName); + results.add(ruleAttributeExpectationDescription); + + return results; + } + + private List createRuleAttributeMail() { + List results = new ArrayList<>(); + RuleAttribute ruleAttributeTitle = new RuleAttribute(); + ruleAttributeTitle.setName("title"); + ruleAttributeTitle.setColumns("B"); + ruleAttributeTitle.setDefaultValue("title"); + + RuleAttribute ruleAttributeDescription = new RuleAttribute(); + ruleAttributeDescription.setName("description"); + ruleAttributeDescription.setColumns("G"); + ruleAttributeDescription.setDefaultValue("description"); + + RuleAttribute ruleAttributeTriggerTime = new RuleAttribute(); + ruleAttributeTriggerTime.setName("trigger_time"); + ruleAttributeTriggerTime.setColumns("C"); + ruleAttributeTriggerTime.setDefaultValue("trigger_time"); + ruleAttributeTriggerTime.setAdditionalConfig(Map.of("timePattern", "")); + + RuleAttribute ruleAttributeMessage = new RuleAttribute(); + ruleAttributeMessage.setName("subject"); + ruleAttributeMessage.setColumns("E"); + ruleAttributeMessage.setDefaultValue("subject"); + + RuleAttribute ruleAttributeSubject = new RuleAttribute(); + ruleAttributeSubject.setName("body"); + ruleAttributeSubject.setColumns("F"); + ruleAttributeSubject.setDefaultValue("body"); + + RuleAttribute ruleAttributeTeams = new RuleAttribute(); + ruleAttributeTeams.setName("teams"); + ruleAttributeTeams.setColumns("D"); + ruleAttributeTeams.setDefaultValue("teams"); + + RuleAttribute ruleAttributeExpectationScore = new RuleAttribute(); + ruleAttributeExpectationScore.setName("expectation_score"); + ruleAttributeExpectationScore.setColumns("J"); + ruleAttributeExpectationScore.setDefaultValue("500"); + + RuleAttribute ruleAttributeExpectationName = new RuleAttribute(); + ruleAttributeExpectationName.setName("expectation_name"); + ruleAttributeExpectationName.setColumns("I"); + ruleAttributeExpectationName.setDefaultValue("name"); + + RuleAttribute ruleAttributeExpectationDescription = new RuleAttribute(); + ruleAttributeExpectationDescription.setName("expectation_description"); + ruleAttributeExpectationDescription.setColumns("H"); + ruleAttributeExpectationDescription.setDefaultValue("description"); + + results.add(ruleAttributeTitle); + results.add(ruleAttributeDescription); + results.add(ruleAttributeTriggerTime); + results.add(ruleAttributeMessage); + results.add(ruleAttributeSubject); + results.add(ruleAttributeTeams); + results.add(ruleAttributeExpectationScore); + results.add(ruleAttributeExpectationName); + results.add(ruleAttributeExpectationDescription); + + return results; + } + + private Object deepCopy(Object objectToCopy, Class classToCopy) throws JsonProcessingException { + ObjectMapper objectMapper = new ObjectMapper(); + + return objectMapper + .readValue(objectMapper.writeValueAsString(objectToCopy), classToCopy); + } +} diff --git a/openbas-api/src/test/java/io/openbas/service/MapperServiceTest.java b/openbas-api/src/test/java/io/openbas/service/MapperServiceTest.java new file mode 100644 index 0000000000..696b909e40 --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/service/MapperServiceTest.java @@ -0,0 +1,201 @@ +package io.openbas.service; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.model.InjectImporter; +import io.openbas.database.repository.ImportMapperRepository; +import io.openbas.database.repository.InjectorContractRepository; +import io.openbas.rest.mapper.form.*; +import io.openbas.utils.mockMapper.MockMapperUtils; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.boot.test.context.SpringBootTest; + +import java.util.Optional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@SpringBootTest +@ExtendWith(MockitoExtension.class) +public class MapperServiceTest { + @Mock + private ImportMapperRepository importMapperRepository; + + @Mock + private InjectorContractRepository injectorContractRepository; + + private MapperService mapperService; + + + + @BeforeEach + void before() { + // Injecting mocks into the controller + mapperService = new MapperService(importMapperRepository, injectorContractRepository); + } + + // -- SCENARIOS -- + + @DisplayName("Test create a mapper") + @Test + void createMapper() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperAddInput importMapperInput = new ImportMapperAddInput(); + importMapperInput.setName(importMapper.getName()); + importMapperInput.setInjectTypeColumn(importMapper.getInjectTypeColumn()); + importMapperInput.setImporters(importMapper.getInjectImporters().stream().map( + injectImporter -> { + InjectImporterAddInput injectImporterAddInput = new InjectImporterAddInput(); + injectImporterAddInput.setInjectTypeValue(injectImporter.getImportTypeValue()); + injectImporterAddInput.setInjectorContractId(injectImporter.getInjectorContract().getId()); + + injectImporterAddInput.setRuleAttributes(injectImporter.getRuleAttributes().stream().map( + ruleAttribute -> { + RuleAttributeAddInput ruleAttributeAddInput = new RuleAttributeAddInput(); + ruleAttributeAddInput.setName(ruleAttribute.getName()); + ruleAttributeAddInput.setColumns(ruleAttribute.getColumns()); + ruleAttributeAddInput.setDefaultValue(ruleAttribute.getDefaultValue()); + ruleAttributeAddInput.setAdditionalConfig(ruleAttribute.getAdditionalConfig()); + return ruleAttributeAddInput; + } + ).toList()); + return injectImporterAddInput; + } + ).toList()); + when(importMapperRepository.save(any())).thenReturn(importMapper); + // -- EXECUTE -- + ImportMapper importMapperResponse = mapperService.createAndSaveImportMapper(importMapperInput); + + // -- ASSERT -- + assertNotNull(importMapperResponse); + assertEquals(importMapperResponse.getId(), importMapper.getId()); + } + + @DisplayName("Test update a specific mapper by using new rule attributes and new inject importer") + @Test + void updateSpecificMapperWithNewElements() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperUpdateInput importMapperInput = new ImportMapperUpdateInput(); + importMapperInput.setName(importMapper.getName()); + importMapperInput.setInjectTypeColumn(importMapper.getInjectTypeColumn()); + importMapperInput.setImporters(importMapper.getInjectImporters().stream().map( + injectImporter -> { + InjectImporterUpdateInput injectImporterUpdateInput = new InjectImporterUpdateInput(); + injectImporterUpdateInput.setInjectTypeValue(injectImporter.getImportTypeValue()); + injectImporterUpdateInput.setInjectorContractId(injectImporter.getInjectorContract().getId()); + + injectImporterUpdateInput.setRuleAttributes(injectImporter.getRuleAttributes().stream().map( + ruleAttribute -> { + RuleAttributeUpdateInput ruleAttributeUpdateInput = new RuleAttributeUpdateInput(); + ruleAttributeUpdateInput.setName(ruleAttribute.getName()); + ruleAttributeUpdateInput.setColumns(ruleAttribute.getColumns()); + ruleAttributeUpdateInput.setDefaultValue(ruleAttribute.getDefaultValue()); + ruleAttributeUpdateInput.setAdditionalConfig(ruleAttribute.getAdditionalConfig()); + return ruleAttributeUpdateInput; + } + ).toList()); + return injectImporterUpdateInput; + } + ).toList()); + when(importMapperRepository.findById(any())).thenReturn(Optional.of(importMapper)); + when(importMapperRepository.save(any())).thenReturn(importMapper); + when(injectorContractRepository.findAllById(any())).thenReturn(importMapper.getInjectImporters().stream().map(InjectImporter::getInjectorContract).toList()); + + // -- EXECUTE -- + ImportMapper response = mapperService.updateImportMapper(importMapper.getId(), importMapperInput); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(response.getId(), importMapper.getId()); + } + + @DisplayName("Test update a specific mapper by creating rule attributes and updating new inject importer") + @Test + void updateSpecificMapperWithUpdatedInjectImporter() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperUpdateInput importMapperInput = new ImportMapperUpdateInput(); + importMapperInput.setName(importMapper.getName()); + importMapperInput.setInjectTypeColumn(importMapper.getInjectTypeColumn()); + importMapperInput.setImporters(importMapper.getInjectImporters().stream().map( + injectImporter -> { + InjectImporterUpdateInput injectImporterUpdateInput = new InjectImporterUpdateInput(); + injectImporterUpdateInput.setInjectTypeValue(injectImporter.getImportTypeValue()); + injectImporterUpdateInput.setInjectorContractId(injectImporter.getInjectorContract().getId()); + injectImporterUpdateInput.setId(injectImporter.getId()); + + injectImporterUpdateInput.setRuleAttributes(injectImporter.getRuleAttributes().stream().map( + ruleAttribute -> { + RuleAttributeUpdateInput ruleAttributeUpdateInput = new RuleAttributeUpdateInput(); + ruleAttributeUpdateInput.setName(ruleAttribute.getName()); + ruleAttributeUpdateInput.setColumns(ruleAttribute.getColumns()); + ruleAttributeUpdateInput.setDefaultValue(ruleAttribute.getDefaultValue()); + ruleAttributeUpdateInput.setAdditionalConfig(ruleAttribute.getAdditionalConfig()); + return ruleAttributeUpdateInput; + } + ).toList()); + return injectImporterUpdateInput; + } + ).toList()); + when(importMapperRepository.findById(any())).thenReturn(Optional.of(importMapper)); + when(importMapperRepository.save(any())).thenReturn(importMapper); + when(injectorContractRepository.findAllById(any())).thenReturn(importMapper.getInjectImporters().stream().map(InjectImporter::getInjectorContract).toList()); + + // -- EXECUTE -- + ImportMapper response = mapperService.updateImportMapper(importMapper.getId(), importMapperInput); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(response.getId(), importMapper.getId()); + } + + @DisplayName("Test update a specific mapper by updating rule attributes and updating inject importer") + @Test + void updateSpecificMapperWithUpdatedElements() throws Exception { + // -- PREPARE -- + ImportMapper importMapper = MockMapperUtils.createImportMapper(); + ImportMapperUpdateInput importMapperInput = new ImportMapperUpdateInput(); + importMapperInput.setName(importMapper.getName()); + importMapperInput.setInjectTypeColumn(importMapper.getInjectTypeColumn()); + importMapperInput.setImporters(importMapper.getInjectImporters().stream().map( + injectImporter -> { + InjectImporterUpdateInput injectImporterUpdateInput = new InjectImporterUpdateInput(); + injectImporterUpdateInput.setInjectTypeValue(injectImporter.getImportTypeValue()); + injectImporterUpdateInput.setInjectorContractId(injectImporter.getInjectorContract().getId()); + injectImporterUpdateInput.setId(injectImporter.getId()); + + injectImporterUpdateInput.setRuleAttributes(injectImporter.getRuleAttributes().stream().map( + ruleAttribute -> { + RuleAttributeUpdateInput ruleAttributeUpdateInput = new RuleAttributeUpdateInput(); + ruleAttributeUpdateInput.setName(ruleAttribute.getName()); + ruleAttributeUpdateInput.setColumns(ruleAttribute.getColumns()); + ruleAttributeUpdateInput.setDefaultValue(ruleAttribute.getDefaultValue()); + ruleAttributeUpdateInput.setAdditionalConfig(ruleAttribute.getAdditionalConfig()); + ruleAttributeUpdateInput.setId(ruleAttribute.getId()); + return ruleAttributeUpdateInput; + } + ).toList()); + return injectImporterUpdateInput; + } + ).toList()); + when(importMapperRepository.findById(any())).thenReturn(Optional.of(importMapper)); + when(importMapperRepository.save(any())).thenReturn(importMapper); + when(injectorContractRepository.findAllById(any())).thenReturn(importMapper.getInjectImporters().stream().map(InjectImporter::getInjectorContract).toList()); + // -- EXECUTE -- + // -- EXECUTE -- + ImportMapper response = mapperService.updateImportMapper(importMapper.getId(), importMapperInput); + + // -- ASSERT -- + assertNotNull(response); + assertEquals(response.getId(), importMapper.getId()); + } + +} diff --git a/openbas-api/src/test/java/io/openbas/utils/CustomMockMultipartFile.java b/openbas-api/src/test/java/io/openbas/utils/CustomMockMultipartFile.java new file mode 100644 index 0000000000..c95b34cb6a --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/utils/CustomMockMultipartFile.java @@ -0,0 +1,20 @@ +package io.openbas.utils; + +import org.springframework.mock.web.MockMultipartFile; + +import java.io.IOException; +import java.io.InputStream; + +//A private inner class, which extends the MockMultipartFile +public class CustomMockMultipartFile extends MockMultipartFile { + + public CustomMockMultipartFile(String name, String originalFilename, String contentType, byte[] content) { + super(name, originalFilename, contentType, content); + } + + //Method is overrided, so that it throws an IOException, when it's called + @Override + public InputStream getInputStream() throws IOException { + throw new IOException(); + } +} diff --git a/openbas-api/src/test/java/io/openbas/utils/mockMapper/MockMapperUtils.java b/openbas-api/src/test/java/io/openbas/utils/mockMapper/MockMapperUtils.java new file mode 100644 index 0000000000..b8fef37af8 --- /dev/null +++ b/openbas-api/src/test/java/io/openbas/utils/mockMapper/MockMapperUtils.java @@ -0,0 +1,50 @@ +package io.openbas.utils.mockMapper; + +import io.openbas.database.model.ImportMapper; +import io.openbas.database.model.InjectImporter; +import io.openbas.database.model.InjectorContract; +import io.openbas.database.model.RuleAttribute; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.Map; +import java.util.UUID; + +public class MockMapperUtils { + public static ImportMapper createImportMapper() { + ImportMapper importMapper = new ImportMapper(); + importMapper.setId(UUID.randomUUID().toString()); + importMapper.setName("Test"); + importMapper.setUpdateDate(Instant.now()); + importMapper.setCreationDate(Instant.now()); + importMapper.setInjectTypeColumn("A"); + importMapper.setInjectImporters(new ArrayList<>()); + + importMapper.getInjectImporters().add(createInjectImporter()); + + return importMapper; + } + + private static InjectImporter createInjectImporter() { + InjectImporter injectImporter = new InjectImporter(); + injectImporter.setId(UUID.randomUUID().toString()); + injectImporter.setImportTypeValue("Test"); + InjectorContract injectorContract = new InjectorContract(); + injectorContract.setId(UUID.randomUUID().toString()); + injectImporter.setInjectorContract(injectorContract); + injectImporter.setRuleAttributes(new ArrayList<>()); + + injectImporter.getRuleAttributes().add(createRuleAttribute()); + return injectImporter; + } + + private static RuleAttribute createRuleAttribute() { + RuleAttribute ruleAttribute = new RuleAttribute(); + ruleAttribute.setColumns("Test"); + ruleAttribute.setName("Test"); + ruleAttribute.setId(UUID.randomUUID().toString()); + ruleAttribute.setAdditionalConfig(Map.of("test", "test")); + ruleAttribute.setDefaultValue(""); + return ruleAttribute; + } +} diff --git a/openbas-api/src/test/resources/xls-test-files/test_file_1.xlsx b/openbas-api/src/test/resources/xls-test-files/test_file_1.xlsx new file mode 100644 index 0000000000..55a0333b1b Binary files /dev/null and b/openbas-api/src/test/resources/xls-test-files/test_file_1.xlsx differ diff --git a/openbas-api/src/test/resources/xls-test-files/test_file_2.xlsx b/openbas-api/src/test/resources/xls-test-files/test_file_2.xlsx new file mode 100644 index 0000000000..e574699951 Binary files /dev/null and b/openbas-api/src/test/resources/xls-test-files/test_file_2.xlsx differ diff --git a/openbas-api/src/test/resources/xls-test-files/test_file_3.xlsx b/openbas-api/src/test/resources/xls-test-files/test_file_3.xlsx new file mode 100644 index 0000000000..99c1750572 Binary files /dev/null and b/openbas-api/src/test/resources/xls-test-files/test_file_3.xlsx differ diff --git a/openbas-api/src/test/resources/xls-test-files/test_file_4.xlsx b/openbas-api/src/test/resources/xls-test-files/test_file_4.xlsx new file mode 100644 index 0000000000..5291106e9f Binary files /dev/null and b/openbas-api/src/test/resources/xls-test-files/test_file_4.xlsx differ diff --git a/openbas-api/src/test/resources/xls-test-files/test_file_5.xlsx b/openbas-api/src/test/resources/xls-test-files/test_file_5.xlsx new file mode 100644 index 0000000000..0cebbca3c0 Binary files /dev/null and b/openbas-api/src/test/resources/xls-test-files/test_file_5.xlsx differ diff --git a/openbas-front/package.json b/openbas-front/package.json index ada1ba8d14..057a42e2d2 100644 --- a/openbas-front/package.json +++ b/openbas-front/package.json @@ -60,6 +60,7 @@ "react-markdown": "9.0.1", "react-redux": "8.1.3", "react-router-dom": "6.23.1", + "react-syntax-highlighter": "15.5.0", "reactflow": "11.11.3", "redux": "4.2.1", "redux-first-history": "5.2.0", @@ -85,6 +86,7 @@ "@types/react": "18.3.3", "@types/react-csv": "1.1.10", "@types/react-dom": "18.3.0", + "@types/react-syntax-highlighter": "15", "@types/seamless-immutable": "7.1.19", "@types/uuid": "9.0.8", "@typescript-eslint/eslint-plugin": "7.10.0", diff --git a/openbas-front/src/actions/InjectorContracts.js b/openbas-front/src/actions/InjectorContracts.js index af54c453ff..9e38856a6d 100644 --- a/openbas-front/src/actions/InjectorContracts.js +++ b/openbas-front/src/actions/InjectorContracts.js @@ -1,11 +1,16 @@ import * as schema from './Schema'; -import { getReferential, putReferential, postReferential, delReferential, simplePostCall } from '../utils/Action'; +import { getReferential, putReferential, postReferential, delReferential, simplePostCall, simpleCall } from '../utils/Action'; export const fetchInjectorContract = (injectorContractId) => (dispatch) => { const uri = `/api/injector_contracts/${injectorContractId}`; return getReferential(schema.injectorContract, uri)(dispatch); }; +export const directFetchInjectorContract = (injectorContractId) => { + const uri = `/api/injector_contracts/${injectorContractId}`; + return simpleCall(uri); +}; + export const fetchInjectorsContracts = () => (dispatch) => { const uri = '/api/injector_contracts'; return getReferential(schema.arrayOfInjectorContracts, uri)(dispatch); diff --git a/openbas-front/src/actions/injector_contracts/InjectorContract.d.ts b/openbas-front/src/actions/injector_contracts/InjectorContract.d.ts index bd3974ef97..fcd3b34194 100644 --- a/openbas-front/src/actions/injector_contracts/InjectorContract.d.ts +++ b/openbas-front/src/actions/injector_contracts/InjectorContract.d.ts @@ -2,4 +2,19 @@ import type { InjectorContract } from '../../utils/api-types'; export type InjectorContractStore = Omit & { injector_contract_attack_patterns: string[] | undefined + injector_contract_injector: string | undefined +}; + +export type ContractType = 'text' | 'number' | 'tuple' | 'checkbox' | 'textarea' | 'select' | 'article' | 'challenge' | 'dependency-select' | 'attachment' | 'team' | 'expectation' | 'asset' | 'asset-group' | 'payload'; + +export interface ContractElement { + key: string; + mandatory: boolean; + type: ContractType; +} + +export type InjectorContractConverted = Omit & { + convertedContent: { + fields: ContractElement[] + } }; diff --git a/openbas-front/src/actions/injector_contracts/injector-contract-helper.d.ts b/openbas-front/src/actions/injector_contracts/injector-contract-helper.d.ts index a9271e0d6a..0fc92161e3 100644 --- a/openbas-front/src/actions/injector_contracts/injector-contract-helper.d.ts +++ b/openbas-front/src/actions/injector_contracts/injector-contract-helper.d.ts @@ -3,7 +3,6 @@ import type { InjectorContract } from '../../utils/api-types'; export interface InjectorContractHelper { getInjectorContract: (injectorContractId: string) => InjectorContract; getInjectorContracts: () => InjectorContract[]; - getInjectorContractsMap: () => Record; getInjectorContractsWithNoTeams: () => Contract['config']['type'][]; getInjectorContractsMapByType: () => Record; } diff --git a/openbas-front/src/actions/mapper/mapper-actions.ts b/openbas-front/src/actions/mapper/mapper-actions.ts new file mode 100644 index 0000000000..0551a5bf39 --- /dev/null +++ b/openbas-front/src/actions/mapper/mapper-actions.ts @@ -0,0 +1,41 @@ +import type { ImportMapperAddInput, ImportMapperUpdateInput, InjectsImportTestInput, RawPaginationImportMapper, SearchPaginationInput } from '../../utils/api-types'; +import { simpleCall, simpleDelCall, simplePostCall, simplePutCall } from '../../utils/Action'; + +const XLS_FORMATTER_URI = '/api/mappers'; + +export const searchMappers = (searchPaginationInput: SearchPaginationInput) => { + const data = searchPaginationInput; + const uri = `${XLS_FORMATTER_URI}/search`; + return simplePostCall(uri, data); +}; + +export const fetchMapper = (mapperId: string) => { + const uri = `${XLS_FORMATTER_URI}/${mapperId}`; + return simpleCall(uri); +}; + +export const deleteMapper = (mapperId: RawPaginationImportMapper['import_mapper_id']) => { + const uri = `${XLS_FORMATTER_URI}/${mapperId}`; + return simpleDelCall(uri, mapperId); +}; + +export const createMapper = (data: ImportMapperAddInput) => { + return simplePostCall(XLS_FORMATTER_URI, data); +}; + +export const updateMapper = (mapperId: string, data: ImportMapperUpdateInput) => { + const uri = `${XLS_FORMATTER_URI}/${mapperId}`; + return simplePutCall(uri, data); +}; + +export const storeXlsFile = (file: File) => { + const uri = `${XLS_FORMATTER_URI}/store`; + const formData = new FormData(); + formData.append('file', file); + return simplePostCall(uri, formData); +}; + +export const testXlsFile = (importId: string, input: InjectsImportTestInput) => { + const uri = `${XLS_FORMATTER_URI}/store/${importId}`; + return simplePostCall(uri, input); +}; diff --git a/openbas-front/src/actions/mapper/mapper.ts b/openbas-front/src/actions/mapper/mapper.ts new file mode 100644 index 0000000000..42a56cba59 --- /dev/null +++ b/openbas-front/src/actions/mapper/mapper.ts @@ -0,0 +1,9 @@ +import type { ImportMapper, InjectImporter } from '../../utils/api-types'; + +export type InjectImporterStore = Omit & { + inject_importer_injector_contract: string; +}; + +export type ImportMapperStore = Omit & { + inject_importers: InjectImporterStore[]; +}; diff --git a/openbas-front/src/actions/scenarios/scenario-actions.ts b/openbas-front/src/actions/scenarios/scenario-actions.ts index a8e054bb81..f5fd3726d3 100644 --- a/openbas-front/src/actions/scenarios/scenario-actions.ts +++ b/openbas-front/src/actions/scenarios/scenario-actions.ts @@ -2,6 +2,7 @@ import { Dispatch } from 'redux'; import { delReferential, getReferential, postReferential, putReferential, simpleCall, simplePostCall } from '../../utils/Action'; import { arrayOfScenarios, scenario } from './scenario-schema'; import type { + InjectsImportInput, Scenario, ScenarioInformationInput, ScenarioInput, @@ -13,6 +14,7 @@ import type { Team, } from '../../utils/api-types'; import * as schema from '../Schema'; +import { MESSAGING$ } from '../../utils/Environment'; const SCENARIO_URI = '/api/scenarios'; @@ -149,3 +151,19 @@ export const fetchScenarioStatistic = () => { const uri = `${SCENARIO_URI}/statistics`; return simpleCall(uri); }; + +// -- IMPORT -- + +export const importXls = (scenarioId: Scenario['scenario_id'], importId: string, input: InjectsImportInput) => { + const uri = `${SCENARIO_URI}/${scenarioId}/xls/${importId}/import`; + return simplePostCall(uri, input) + .then((response) => { + const injectCount = response.data.injects.length; + if (injectCount === 0) { + MESSAGING$.notifySuccess('No inject imported'); + } else { + MESSAGING$.notifySuccess(`${injectCount} inject imported`); + } + return response; + }); +}; diff --git a/openbas-front/src/admin/components/atomic_testings/atomic_testing/AtomicTestingPopover.tsx b/openbas-front/src/admin/components/atomic_testings/atomic_testing/AtomicTestingPopover.tsx index b8b96d6bdb..17ee444ab8 100644 --- a/openbas-front/src/admin/components/atomic_testings/atomic_testing/AtomicTestingPopover.tsx +++ b/openbas-front/src/admin/components/atomic_testings/atomic_testing/AtomicTestingPopover.tsx @@ -4,7 +4,7 @@ import { useNavigate } from 'react-router-dom'; import type { Inject, InjectResultDTO } from '../../../../utils/api-types'; import { useFormatter } from '../../../../components/i18n'; import { useAppDispatch } from '../../../../utils/hooks'; -import ButtonPopover, { ButtonPopoverEntry, VariantButtonPopover } from '../../../../components/common/ButtonPopover'; +import ButtonPopover, { PopoverEntry, VariantButtonPopover } from '../../../../components/common/ButtonPopover'; import DialogDelete from '../../../../components/common/DialogDelete'; import { deleteAtomicTesting, duplicateAtomicTesting, updateAtomicTesting } from '../../../../actions/atomic_testings/atomic-testing-actions'; import { useHelper } from '../../../../store'; @@ -19,7 +19,7 @@ import { isNotEmptyField } from '../../../../utils/utils'; interface Props { atomic: InjectResultDTO; - entries: ButtonPopoverEntry[]; + entries: PopoverEntry[]; openEdit?: boolean; openDelete?: boolean; openDuplicate?: boolean; diff --git a/openbas-front/src/admin/components/common/Context.ts b/openbas-front/src/admin/components/common/Context.ts index 80fed2c764..7a28a8e161 100644 --- a/openbas-front/src/admin/components/common/Context.ts +++ b/openbas-front/src/admin/components/common/Context.ts @@ -1,6 +1,6 @@ import { createContext } from 'react'; import type { ArticleStore, FullArticleStore } from '../../../actions/channels/Article'; -import type { ArticleCreateInput, ArticleUpdateInput, Inject, Team, TeamCreateInput, Variable, VariableInput } from '../../../utils/api-types'; +import type { ArticleCreateInput, ArticleUpdateInput, Inject, InjectsImportInput, Team, TeamCreateInput, Variable, VariableInput } from '../../../utils/api-types'; import type { UserStore } from '../teams/players/Player'; import type { InjectStore } from '../../../actions/injects/Inject'; @@ -54,6 +54,7 @@ export type InjectContextType = { }>, onInjectDone?: (injectId: Inject['inject_id']) => void, onDeleteInject: (injectId: Inject['inject_id']) => void, + onImportInjectFromXls?: (importId: string, input: InjectsImportInput) => Promise }; export type AtomicTestingContextType = { @@ -129,6 +130,10 @@ export const InjectContext = createContext({ }, onDeleteInject(_injectId: Inject['inject_id']): void { }, + onImportInjectFromXls(_importId: string, _input: InjectsImportInput): Promise { + return new Promise(() => { + }); + }, }); export const AtomicTestingContext = createContext({ diff --git a/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromInjectsTest.tsx b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromInjectsTest.tsx new file mode 100644 index 0000000000..359e0baaa8 --- /dev/null +++ b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromInjectsTest.tsx @@ -0,0 +1,176 @@ +import { Autocomplete as MuiAutocomplete, Box, Button, MenuItem, TextField } from '@mui/material'; +import React, { FunctionComponent, SyntheticEvent, useState } from 'react'; +import { Controller, useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { z } from 'zod'; +import moment from 'moment-timezone'; +import { makeStyles } from '@mui/styles'; +import { zodImplement } from '../../../../utils/Zod'; +import { useFormatter } from '../../../../components/i18n'; +import type { ImportMapperAddInput, ImportTestSummary, InjectsImportTestInput } from '../../../../utils/api-types'; +import CodeBlock from '../../../../components/common/CodeBlock'; +import { testXlsFile } from '../../../../actions/mapper/mapper-actions'; + +const useStyles = makeStyles(() => ({ + container: { + display: 'flex', + flexDirection: 'column', + gap: '16px', + }, + buttons: { + display: 'flex', + justifyContent: 'right', + gap: '8px', + marginTop: '24px', + }, +})); + +interface FormProps { + sheetName: string; + timezone: string; +} + +interface Props { + importId: string; + sheets: string[]; + importMapperValues: ImportMapperAddInput; + handleClose: () => void; +} + +const ImportUploaderInjectFromInjectsTest: FunctionComponent = ({ + importId, + sheets, + importMapperValues, + handleClose, +}) => { + // Standard hooks + const { t } = useFormatter(); + const classes = useStyles(); + + // TimeZone + const timezones = moment.tz.names(); + + // Form + const { + register, + control, + handleSubmit: handleSubmitForm, + formState: { errors, isDirty, isSubmitting }, + } = useForm({ + mode: 'onTouched', + resolver: zodResolver( + zodImplement().with({ + sheetName: z.string().min(1, { message: t('Should not be empty') }), + timezone: z.string().min(1, { message: t('Should not be empty') }), + }), + ), + defaultValues: { + timezone: moment.tz.guess(), + }, + }); + + const [result, setResult] = useState(undefined); + + const onSubmitImportTest = (values: FormProps) => { + const input: InjectsImportTestInput = { + import_mapper: importMapperValues, + sheet_name: values.sheetName, + timezone_offset: moment.tz(values.timezone).utcOffset(), + }; + testXlsFile(importId, input).then((rs: { data: ImportTestSummary }) => { + const { data } = rs; + setResult(data); + }); + }; + + const handleSubmitWithoutPropagation = (e: SyntheticEvent) => { + e.preventDefault(); + e.stopPropagation(); + handleSubmitForm(onSubmitImportTest)(e); + }; + + return ( +
+
+ ( + { + onChange(v); + }} + renderInput={(params) => ( + + )} + /> + )} + /> + ( + {timezones.map((tz) => ( + {t(tz)} + ))} + + )} + /> +
+ + {t('Result')} : + + + + {t('Log')} : + i.message_level === 'ERROR'), null, ' ') || t('You will find here the result log in JSON format.')} + language={'json'} + /> + +
+ + +
+
+ ); +}; + +export default ImportUploaderInjectFromInjectsTest; diff --git a/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXls.tsx b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXls.tsx new file mode 100644 index 0000000000..351af05dd8 --- /dev/null +++ b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXls.tsx @@ -0,0 +1,87 @@ +import { ToggleButton, Tooltip } from '@mui/material'; +import { CloudUploadOutlined } from '@mui/icons-material'; +import React, { useContext, useState } from 'react'; +import Dialog from '../../../../components/common/Dialog'; +import { useFormatter } from '../../../../components/i18n'; +import type { ImportPostSummary, InjectsImportInput } from '../../../../utils/api-types'; +import ImportUploaderInjectFromXlsFile from './ImportUploaderInjectFromXlsFile'; +import ImportUploaderInjectFromXlsInjects from './ImportUploaderInjectFromXlsInjects'; +import { InjectContext } from '../Context'; +import { storeXlsFile } from '../../../../actions/mapper/mapper-actions'; + +const ImportUploaderInjectFromXls = () => { + // Standard hooks + const { t } = useFormatter(); + const injectContext = useContext(InjectContext); + + const [importId, setImportId] = useState(undefined); + const [sheets, setSheets] = useState([]); + + // Dialog + const [open, setOpen] = useState(false); + const handleOpen = () => setOpen(true); + const handleClose = () => { + setImportId(undefined); + setSheets([]); + setOpen(false); + }; + + const onSubmitImportFile = (values: { file: File }) => { + storeXlsFile(values.file).then((result: { data: ImportPostSummary }) => { + const { data } = result; + setImportId(data.import_id); + setSheets(data.available_sheets); + }); + }; + + const onSubmitImportInjects = (input: InjectsImportInput) => { + if (importId) { + injectContext.onImportInjectFromXls?.(importId, input).then(() => { + handleClose(); + }); + } + }; + + return ( + <> + + + + + + + <> + {!importId + && + } + {importId + && + } + + + + ); +}; + +export default ImportUploaderInjectFromXls; diff --git a/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsFile.tsx b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsFile.tsx new file mode 100644 index 0000000000..5d83ef234e --- /dev/null +++ b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsFile.tsx @@ -0,0 +1,112 @@ +import { Button } from '@mui/material'; +import React, { FunctionComponent, SyntheticEvent, useState } from 'react'; +import { Controller, useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { z } from 'zod'; +import { makeStyles } from '@mui/styles'; +import { zodImplement } from '../../../../utils/Zod'; +import { useFormatter } from '../../../../components/i18n'; +import CustomFileUploader from '../../../../components/common/CustomFileUploader'; +import Loader from '../../../../components/Loader'; + +const useStyles = makeStyles(() => ({ + container: { + display: 'flex', + flexDirection: 'column', + gap: '16px', + }, + buttons: { + display: 'flex', + justifyContent: 'right', + gap: '8px', + marginTop: '24px', + }, +})); + +interface FormProps { + file: File; +} + +interface Props { + handleClose: () => void; + handleSubmit: (values: FormProps) => void; +} + +const ImportUploaderInjectFromXlsFile: FunctionComponent = ({ + handleClose, + handleSubmit, +}) => { + // Standard hooks + const { t } = useFormatter(); + const classes = useStyles(); + + const [loading, setLoading] = useState(false); + + // Form + const { + control, + handleSubmit: handleSubmitForm, + formState: { errors, isDirty, isSubmitting }, + } = useForm({ + mode: 'onTouched', + resolver: zodResolver( + zodImplement().with({ + file: z.instanceof(File), + }), + ), + }); + + const onSubmitImportFile = (values: FormProps) => { + setLoading(true); + handleSubmit(values); + setLoading(false); + }; + + const handleSubmitWithoutPropagation = (e: SyntheticEvent) => { + e.preventDefault(); + e.stopPropagation(); + handleSubmitForm(onSubmitImportFile)(e); + }; + + return ( + <> + {loading && } + {!loading + &&
+
+ ( + + )} + /> +
+
+ + +
+
+ } + + ); +}; + +export default ImportUploaderInjectFromXlsFile; diff --git a/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsInjects.tsx b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsInjects.tsx new file mode 100644 index 0000000000..1ebb648ae2 --- /dev/null +++ b/openbas-front/src/admin/components/common/injects/ImportUploaderInjectFromXlsInjects.tsx @@ -0,0 +1,220 @@ +import { Autocomplete as MuiAutocomplete, Box, Button, MenuItem, TextField } from '@mui/material'; +import { TableViewOutlined } from '@mui/icons-material'; +import React, { FunctionComponent, SyntheticEvent, useEffect, useState } from 'react'; +import { Controller, useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { z } from 'zod'; +import moment from 'moment-timezone'; +import { makeStyles } from '@mui/styles'; +import { zodImplement } from '../../../../utils/Zod'; +import { useFormatter } from '../../../../components/i18n'; +import type { ImportMapper, InjectsImportInput } from '../../../../utils/api-types'; +import { searchMappers } from '../../../../actions/mapper/mapper-actions'; +import type { Page } from '../../../../components/common/pagination/Page'; + +const useStyles = makeStyles(() => ({ + container: { + display: 'flex', + flexDirection: 'column', + gap: '16px', + }, + buttons: { + display: 'flex', + justifyContent: 'right', + gap: '8px', + marginTop: '24px', + }, + icon: { + paddingTop: 4, + display: 'inline-block', + }, + text: { + display: 'inline-block', + flexGrow: 1, + marginLeft: 10, + }, +})); + +interface FormProps { + sheetName: string; + importMapperId: string; + timezone: string; +} + +interface Props { + sheets: string[]; + handleClose: () => void; + handleSubmit: (input: InjectsImportInput) => void; +} + +const ImportUploaderInjectFromXlsInjects: FunctionComponent = ({ + sheets, + handleClose, + handleSubmit, +}) => { + // Standard hooks + const { t } = useFormatter(); + const classes = useStyles(); + + // TimeZone + const timezones = moment.tz.names(); + + // Form + const { + register, + control, + handleSubmit: handleSubmitForm, + formState: { errors, isDirty, isSubmitting }, + } = useForm({ + mode: 'onTouched', + resolver: zodResolver( + zodImplement().with({ + sheetName: z.string().min(1, { message: t('Should not be empty') }), + importMapperId: z.string().min(1, { message: t('Should not be empty') }), + timezone: z.string().min(1, { message: t('Should not be empty') }), + }), + ), + defaultValues: { + timezone: moment.tz.guess(), + }, + }); + + // Mapper + const [mappers, setMappers] = useState([]); + useEffect(() => { + searchMappers({ size: 10 }).then((result: { data: Page }) => { + const { data } = result; + setMappers(data.content); + }); + }, []); + const mapperOptions = mappers.map( + (m) => ({ + id: m.import_mapper_id, + label: m.import_mapper_name, + }), + ); + + const onSubmitImportInjects = (values: FormProps) => { + const input: InjectsImportInput = { + import_mapper_id: values.importMapperId, + sheet_name: values.sheetName, + timezone_offset: moment.tz(values.timezone).utcOffset(), + }; + handleSubmit(input); + }; + + const handleSubmitWithoutPropagation = (e: SyntheticEvent) => { + e.preventDefault(); + e.stopPropagation(); + handleSubmitForm(onSubmitImportInjects)(e); + }; + + return ( +
+
+ ( + { + onChange(v); + }} + renderInput={(params) => ( + + )} + /> + )} + /> + ( + { + onChange(v?.id); + }} + renderOption={(props, option) => ( + +
+ +
+
{option.label}
+ + )} + getOptionLabel={(option) => option.label} + isOptionEqualToValue={(option, v) => option.id === v.id} + renderInput={(params) => ( + + )} + /> + )} + /> + ( + {timezones.map((tz) => ( + {t(tz)} + ))} + + )} + /> +
+
+ + +
+
+ ); +}; + +export default ImportUploaderInjectFromXlsInjects; diff --git a/openbas-front/src/admin/components/common/injects/Injects.js b/openbas-front/src/admin/components/common/injects/Injects.js index 6bf602d5dd..2b64b4a5f0 100644 --- a/openbas-front/src/admin/components/common/injects/Injects.js +++ b/openbas-front/src/admin/components/common/injects/Injects.js @@ -35,6 +35,7 @@ import UpdateInject from './UpdateInject'; import PlatformIcon from '../../../../components/PlatformIcon'; import Timeline from '../../../../components/Timeline'; import { isNotEmptyField } from '../../../../utils/utils'; +import ImportUploaderInjectFromXls from './ImportUploaderInjectFromXls'; import useExportToXLS from '../../../../utils/hooks/useExportToXLS'; const useStyles = makeStyles(() => ({ @@ -301,14 +302,16 @@ const Injects = (props) => { )} - {setViewMode ? ( - - + + {injectContext.onImportInjectFromXls + && } + {setViewMode + && { > - - + + } + {setViewMode + && setViewMode('distribution')} @@ -325,9 +330,9 @@ const Injects = (props) => { > - - - ) : null} + + } +
diff --git a/openbas-front/src/admin/components/nav/LeftBar.tsx b/openbas-front/src/admin/components/nav/LeftBar.tsx index 5d08b037f5..0e6a45103d 100644 --- a/openbas-front/src/admin/components/nav/LeftBar.tsx +++ b/openbas-front/src/admin/components/nav/LeftBar.tsx @@ -514,10 +514,10 @@ const LeftBar = () => { {navOpen && ( - + )} @@ -643,6 +643,7 @@ const LeftBar = () => { { link: '/admin/settings', label: 'Parameters', exact: true }, { link: '/admin/settings/security', label: 'Security' }, { link: '/admin/settings/taxonomies', label: 'Taxonomies' }, + { link: '/admin/settings/data_ingestion', label: 'Data ingestion' }, ], )} diff --git a/openbas-front/src/admin/components/scenarios/scenario/ScenarioContext.ts b/openbas-front/src/admin/components/scenarios/scenario/ScenarioContext.ts index a08d7dfdf2..c1cd28d25d 100644 --- a/openbas-front/src/admin/components/scenarios/scenario/ScenarioContext.ts +++ b/openbas-front/src/admin/components/scenarios/scenario/ScenarioContext.ts @@ -1,8 +1,9 @@ -import type { Inject } from '../../../../utils/api-types'; +import type { Inject, InjectsImportInput } from '../../../../utils/api-types'; import { addInjectForScenario, deleteInjectScenario, updateInjectActivationForScenario, updateInjectForScenario } from '../../../../actions/Inject'; import { useAppDispatch } from '../../../../utils/hooks'; import type { ScenarioStore } from '../../../../actions/scenarios/Scenario'; import type { InjectStore } from '../../../../actions/injects/Inject'; +import { importXls } from '../../../../actions/scenarios/scenario-actions'; const injectContextForScenario = (scenario: ScenarioStore) => { const dispatch = useAppDispatch(); @@ -23,6 +24,10 @@ const injectContextForScenario = (scenario: ScenarioStore) => { onDeleteInject(injectId: Inject['inject_id']): void { return dispatch(deleteInjectScenario(scenario.scenario_id, injectId)); }, + onImportInjectFromXls(importId: string, input: InjectsImportInput): Promise { + // @ts-expect-error: handle axios + return importXls(scenario.scenario_id, importId, input); + }, }; }; diff --git a/openbas-front/src/admin/components/scenarios/scenario/ScenarioHeader.tsx b/openbas-front/src/admin/components/scenarios/scenario/ScenarioHeader.tsx index 38c214d452..a48907de75 100644 --- a/openbas-front/src/admin/components/scenarios/scenario/ScenarioHeader.tsx +++ b/openbas-front/src/admin/components/scenarios/scenario/ScenarioHeader.tsx @@ -15,7 +15,8 @@ import { parseCron, ParsedCron } from '../../../../utils/Cron'; import ScenarioRecurringFormDialog from './ScenarioRecurringFormDialog'; import { truncate } from '../../../../utils/String'; import type { Theme } from '../../../../components/Theme'; -import { ButtonPopoverEntry } from '../../../../components/common/ButtonPopover'; +import { PopoverEntry } from '../../../../components/common/ButtonPopover'; +import useScenarioPermissions from '../../../../utils/Scenario'; const useStyles = makeStyles(() => ({ title: { @@ -149,12 +150,14 @@ const ScenarioHeader = ({ setOpenDeleteId(null); }; + const permissions = useScenarioPermissions(scenario.scenario_id); + // Button Popover - const entries: ButtonPopoverEntry[] = [ - { label: 'Update', action: () => handleOpenEdit() }, + const entries: PopoverEntry[] = [ + { label: 'Update', action: () => handleOpenEdit(), disabled: !permissions.canWrite }, { label: 'Duplicate', action: () => handleOpenDuplicate() }, { label: 'Export', action: () => handleOpenExport() }, - { label: 'Delete', action: () => handleOpenDelete() }, + { label: 'Delete', action: () => handleOpenDelete(), disabled: !permissions.canWrite }, ]; return ( diff --git a/openbas-front/src/admin/components/scenarios/scenario/ScenarioPopover.tsx b/openbas-front/src/admin/components/scenarios/scenario/ScenarioPopover.tsx index 6ea80425d3..58c4ebe64b 100644 --- a/openbas-front/src/admin/components/scenarios/scenario/ScenarioPopover.tsx +++ b/openbas-front/src/admin/components/scenarios/scenario/ScenarioPopover.tsx @@ -5,8 +5,8 @@ import type { ScenarioInput, ScenarioInformationInput } from '../../../../utils/ import { useFormatter } from '../../../../components/i18n'; import { useAppDispatch } from '../../../../utils/hooks'; import type { ScenarioStore } from '../../../../actions/scenarios/Scenario'; -import { deleteScenario, duplicateScenario, exportScenarioUri, updateScenario, updateScenarioInformation } from '../../../../actions/scenarios/scenario-actions'; -import ButtonPopover, { ButtonPopoverEntry, VariantButtonPopover } from '../../../../components/common/ButtonPopover'; +import { deleteScenario, exportScenarioUri, updateScenario, updateScenarioInformation, duplicateScenario } from '../../../../actions/scenarios/scenario-actions'; +import ButtonPopover, { PopoverEntry, VariantButtonPopover } from '../../../../components/common/ButtonPopover'; import Drawer from '../../../../components/common/Drawer'; import ScenarioForm from '../ScenarioForm'; import DialogDelete from '../../../../components/common/DialogDelete'; @@ -18,7 +18,7 @@ import DialogDuplicate from '../../../../components/common/DialogDuplicate'; interface Props { scenario: ScenarioStore; - entries: ButtonPopoverEntry[]; + entries: PopoverEntry[]; openEdit?: boolean; setOpenEdit?: React.Dispatch>; openExport?: boolean; diff --git a/openbas-front/src/admin/components/settings/DataIngestionMenu.tsx b/openbas-front/src/admin/components/settings/DataIngestionMenu.tsx new file mode 100644 index 0000000000..fa337ee848 --- /dev/null +++ b/openbas-front/src/admin/components/settings/DataIngestionMenu.tsx @@ -0,0 +1,18 @@ +import React from 'react'; +import { TableViewOutlined } from '@mui/icons-material'; +import RightMenu, { RightMenuEntry } from '../../../components/common/RightMenu'; + +const DataIngestionMenu = () => { + const entries: RightMenuEntry[] = [ + { + path: '/admin/settings/data_ingestion/xls_formatter', + icon: () => (), + label: 'Xls formatters', + }, + ]; + return ( + + ); +}; + +export default DataIngestionMenu; diff --git a/openbas-front/src/admin/components/settings/Index.tsx b/openbas-front/src/admin/components/settings/Index.tsx index 39776936be..5936da577b 100644 --- a/openbas-front/src/admin/components/settings/Index.tsx +++ b/openbas-front/src/admin/components/settings/Index.tsx @@ -1,5 +1,6 @@ import React from 'react'; import { Navigate, Route, Routes } from 'react-router-dom'; +import XlsFormatters from './data_ingestion/XlsFormatters'; import Parameters from './Parameters'; import Users from './users/Users'; import Groups from './groups/Groups'; @@ -21,8 +22,10 @@ const Index = () => ( + } /> + {/* Not found */} - }/> + } /> ); diff --git a/openbas-front/src/admin/components/settings/data_ingestion/AttributeUtils.ts b/openbas-front/src/admin/components/settings/data_ingestion/AttributeUtils.ts new file mode 100644 index 0000000000..1c0a6af90c --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/AttributeUtils.ts @@ -0,0 +1,14 @@ +const alphabet = (size = 0) => { + const fn = () => Array.from(Array(26)) + .map((_, i) => i + 65) + .map((x) => String.fromCharCode(x)); + const letters: string[] = fn(); + for (let step = 0; step < size; step += 1) { + const additionalLetters = fn(); + const firstLetter = additionalLetters[step]; + letters.push(...additionalLetters.map((l) => firstLetter.concat(l))); + } + return letters; +}; + +export default alphabet; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/XlsFormatters.tsx b/openbas-front/src/admin/components/settings/data_ingestion/XlsFormatters.tsx new file mode 100644 index 0000000000..8a60f11059 --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/XlsFormatters.tsx @@ -0,0 +1,143 @@ +import React, { CSSProperties, useState } from 'react'; +import { makeStyles } from '@mui/styles'; +import { List, ListItem, ListItemIcon, ListItemSecondaryAction, ListItemText } from '@mui/material'; +import { TableViewOutlined } from '@mui/icons-material'; +import { useFormatter } from '../../../../components/i18n'; +import Breadcrumbs from '../../../../components/Breadcrumbs'; +import SortHeadersComponent from '../../../../components/common/pagination/SortHeadersComponent'; +import type { RawPaginationImportMapper, SearchPaginationInput } from '../../../../utils/api-types'; +import { searchMappers } from '../../../../actions/mapper/mapper-actions'; +import { initSorting } from '../../../../components/common/pagination/Page'; +import Empty from '../../../../components/Empty'; +import DataIngestionMenu from '../DataIngestionMenu'; +import XlsFormatterCreation from './xls_formatter/XlsFormatterCreation'; +import PaginationComponent from '../../../../components/common/pagination/PaginationComponent'; +import XlsMapperPopover from './XlsMapperPopover'; + +const useStyles = makeStyles(() => ({ + container: { + padding: '0 200px 50px 0', + }, + itemHead: { + paddingLeft: 10, + textTransform: 'uppercase', + cursor: 'pointer', + }, + item: { + paddingLeft: 10, + height: 50, + }, + bodyItems: { + display: 'flex', + alignItems: 'center', + }, + bodyItem: { + height: 20, + fontSize: 13, + whiteSpace: 'nowrap', + overflow: 'hidden', + textOverflow: 'ellipsis', + paddingRight: 10, + }, +})); + +const inlineStyles: Record = { + mapper_name: { + width: '30%', + cursor: 'default', + }, +}; + +const XlsFormatters = () => { + // Standard hooks + const classes = useStyles(); + const { t } = useFormatter(); + + // Headers + const headers = [ + { + field: 'mapper_name', + label: 'Name', + isSortable: true, + value: (mapper: RawPaginationImportMapper) => mapper.import_mapper_name, + }, + ]; + + const [mappers, setMappers] = useState([]); + const [searchPaginationInput, setSearchPaginationInput] = useState({ + sorts: initSorting('import_mapper_name'), + }); + + return ( +
+ + + + + + + + } + /> + + { + mappers.map((mapper) => { + return ( + + + + + + {headers.map((header) => ( +
+ {header.value(mapper)} +
+ ))} +
+ } + /> + + setMappers(mappers.map((existing) => (existing.import_mapper_id !== result.import_mapper_id ? existing : result)))} + onDelete={(result) => setMappers(mappers.filter((existing) => (existing.import_mapper_id !== result)))} + /> + + + ); + }) + } + {!mappers ? () : null} + + setMappers([result, ...mappers])} /> + + ); +}; + +export default XlsFormatters; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/XlsMapperPopover.tsx b/openbas-front/src/admin/components/settings/data_ingestion/XlsMapperPopover.tsx new file mode 100644 index 0000000000..ed6d427d29 --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/XlsMapperPopover.tsx @@ -0,0 +1,73 @@ +import React, { FunctionComponent, useState } from 'react'; +import { PopoverEntry } from '../../../../components/common/ButtonPopover'; +import IconPopover from '../../../../components/common/IconPopover'; +import type { RawPaginationImportMapper } from '../../../../utils/api-types'; +import { deleteMapper } from '../../../../actions/mapper/mapper-actions'; +import DialogDelete from '../../../../components/common/DialogDelete'; +import { useFormatter } from '../../../../components/i18n'; +import Drawer from '../../../../components/common/Drawer'; +import XlsFormatterUpdate from './xls_formatter/XlsFormatterUpdate'; + +interface Props { + mapper: RawPaginationImportMapper; + onUpdate?: (result: RawPaginationImportMapper) => void; + onDelete?: (result: string) => void; +} + +const XlsMapperPopover: FunctionComponent = ({ + mapper, + onUpdate, + onDelete, +}) => { + // Standard hooks + const { t } = useFormatter(); + + // Edition + const [openEdit, setOpenEdit] = useState(false); + + const handleOpenEdit = () => setOpenEdit(true); + const handleCloseEdit = () => setOpenEdit(false); + + // Deletion + const [openDelete, setOpenDelete] = useState(false); + + const handleOpenDelete = () => setOpenDelete(true); + const handleCloseDelete = () => setOpenDelete(false); + const submitDelete = () => { + deleteMapper(mapper.import_mapper_id); + if (onDelete) { + onDelete(mapper.import_mapper_id); + } + handleCloseDelete(); + }; + + const entries: PopoverEntry[] = [ + { label: 'Update', action: handleOpenEdit }, + { label: 'Delete', action: handleOpenDelete }, + ]; + + return ( + <> + + + + + + + ); +}; + +export default XlsMapperPopover; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/MapperForm.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/MapperForm.tsx new file mode 100644 index 0000000000..efcdf9d11f --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/MapperForm.tsx @@ -0,0 +1,168 @@ +import { Controller, SubmitHandler, useFieldArray, useForm } from 'react-hook-form'; +import React, { useState } from 'react'; +import { Button, IconButton, TextField, Typography } from '@mui/material'; +import { Add } from '@mui/icons-material'; +import { makeStyles } from '@mui/styles'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { z } from 'zod'; +import type { ImportMapperAddInput } from '../../../../../utils/api-types'; +import { useFormatter } from '../../../../../components/i18n'; +import { zodImplement } from '../../../../../utils/Zod'; +import RegexComponent from '../../../../../components/RegexComponent'; +import RulesContractContent from './RulesContractContent'; +import XlsMapperTestDialog from './XlsMapperTestDialog'; + +const useStyles = makeStyles(() => ({ + importerStyle: { + display: 'flex', + alignItems: 'center', + marginTop: 20, + }, + importersErrorMessage: { + fontSize: 13, + color: '#f44336', + }, +})); + +interface Props { + onSubmit: SubmitHandler; + editing?: boolean; + initialValues?: ImportMapperAddInput; +} + +const MapperForm: React.FC = ({ + onSubmit, + editing, + initialValues = { + mapper_name: '', + mapper_inject_type_column: '', + mapper_inject_importers: [], + }, +}) => { + // Standard hooks + const { t } = useFormatter(); + const classes = useStyles(); + + const ruleAttributeZodObject = z.object({ + rule_attribute_name: z.string().min(1, { message: t('Should not be empty') }), + rule_attribute_columns: z.string().optional().nullable(), + rule_attribute_default_value: z.string().optional(), + rule_attribute_additional_config: z.record(z.string(), z.string()).optional(), + }); + + const importerZodObject = z.object({ + inject_importer_type_value: z.string().min(1, { message: t('Should not be empty') }), + inject_importer_injector_contract_id: z.string().min(1, { message: t('Should not be empty') }), + inject_importer_rule_attributes: z.array(ruleAttributeZodObject).optional(), + }); + + const methods = useForm({ + mode: 'onTouched', + resolver: zodResolver( + zodImplement().with({ + mapper_name: z.string().min(1, { message: t('Should not be empty') }), + mapper_inject_importers: z.array(importerZodObject) + .min(1, { message: t('At least one inject type is required') }), + mapper_inject_type_column: z.string() + .min(1, { message: t('Should not be empty') }), + }), + ), + defaultValues: initialValues, + }); + + const { control, getValues } = methods; + + const { fields, append, remove } = useFieldArray({ + control, + name: 'mapper_inject_importers', + }); + + const [openTest, setOpenTest] = useState(false); + + return ( + <> +
+ +
+ ( + + )} + /> +
+
+ + {t('Representation for inject type')} + + { + append({ inject_importer_type_value: '', inject_importer_injector_contract_id: '', inject_importer_rule_attributes: [] }); + }} + size="large" + > + + +
+ {methods.formState.errors.mapper_inject_importers?.message} +
+
+ + {fields.map((field, index) => ( + + ))} + +
+ + +
+ + setOpenTest(false)} + importMapperValues={getValues()} + /> + + ); +}; + +export default MapperForm; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/RulesContractContent.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/RulesContractContent.tsx new file mode 100644 index 0000000000..4a0463b0af --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/RulesContractContent.tsx @@ -0,0 +1,353 @@ +import { Controller, FieldArrayWithId, useFieldArray, UseFieldArrayRemove, UseFormReturn } from 'react-hook-form'; +import { + Accordion, + AccordionActions, + AccordionDetails, + AccordionSummary, + Badge, + Button, + Dialog, + DialogActions, + DialogContent, + DialogContentText, + DialogTitle, + IconButton, + TextField, + Tooltip, + Typography, +} from '@mui/material'; +import { DeleteOutlined, ExpandMore } from '@mui/icons-material'; +import { CogOutline, InformationOutline } from 'mdi-material-ui'; +import React, { useEffect, useState } from 'react'; +import { makeStyles } from '@mui/styles'; +import classNames from 'classnames'; +import { directFetchInjectorContract } from '../../../../../actions/InjectorContracts'; +import InjectContractComponent from '../../../../../components/InjectContractComponent'; +import { useFormatter } from '../../../../../components/i18n'; +import RegexComponent from '../../../../../components/RegexComponent'; +import type { ImportMapperAddInput } from '../../../../../utils/api-types'; +import type { ContractElement, InjectorContractConverted } from '../../../../../actions/injector_contracts/InjectorContract'; + +const useStyles = makeStyles(() => ({ + rulesArray: { + gap: '10px', + width: '100%', + display: 'inline-grid', + marginTop: '10px', + alignItems: 'center', + gridTemplateColumns: ' 1fr 3fr 50px', + }, + container: { + display: 'inline-flex', + alignItems: 'center', + }, + redStar: { + color: 'rgb(244, 67, 54)', + marginLeft: '2px', + }, + red: { + borderColor: 'rgb(244, 67, 54)', + }, +})); + +interface Props { + field: FieldArrayWithId; + methods: UseFormReturn; + index: number; + remove: UseFieldArrayRemove; +} + +const RulesContractContent: React.FC = ({ + field, + methods, + index, + remove, +}) => { + const { t, tPick } = useFormatter(); + const classes = useStyles(); + + // Fetching data + + const { control, formState: { errors } } = methods; + + const { fields: rulesFields, remove: rulesRemove, append: rulesAppend } = useFieldArray({ + control, + name: `mapper_inject_importers.${index}.inject_importer_rule_attributes`, + }); + + const [contractFields, setContractFields] = useState([]); + const [injectorContractLabel, setInjectorContractLabel] = useState(undefined); + + const isMandatoryField = (fieldKey: string) => { + return ['title'].includes(fieldKey) || contractFields.find((f) => f.key === fieldKey)?.mandatory; + }; + + const addRules = (contractFieldKeys: string[]) => { + rulesAppend({ + rule_attribute_name: 'title', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + rulesAppend({ + rule_attribute_name: 'description', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + rulesAppend({ + rule_attribute_name: 'trigger_time', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + // eslint-disable-next-line no-plusplus + for (let i = 0; i < contractFieldKeys?.length; i++) { + rulesAppend({ + rule_attribute_name: contractFieldKeys[i], + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + } + rulesAppend({ + rule_attribute_name: 'expectation_name', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + rulesAppend({ + rule_attribute_name: 'expectation_description', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + rulesAppend({ + rule_attribute_name: 'expectation_score', + rule_attribute_columns: '', + rule_attribute_default_value: '', + }); + }; + + useEffect(() => { + if (methods.getValues(`mapper_inject_importers.${index}.inject_importer_injector_contract_id`)) { + directFetchInjectorContract(methods.getValues(`mapper_inject_importers.${index}.inject_importer_injector_contract_id`)).then((result: { + data: InjectorContractConverted + }) => { + const injectorContract = result.data; + setInjectorContractLabel(tPick(injectorContract.injector_contract_labels)); + const tmp = injectorContract?.convertedContent?.fields + .filter((f) => !['checkbox', 'attachment', 'expectation'].includes(f.type)); + setContractFields(tmp); + }); + } + }, []); + + const onChangeInjectorContractId = () => { + directFetchInjectorContract(methods.getValues(`mapper_inject_importers.${index}.inject_importer_injector_contract_id`)).then((result: { data: InjectorContractConverted }) => { + const injectorContract = result.data; + setInjectorContractLabel(tPick(injectorContract.injector_contract_labels)); + const tmp = injectorContract?.convertedContent?.fields + .filter((f) => !['checkbox', 'attachment', 'expectation'].includes(f.type)); + setContractFields(tmp); + const contractFieldKeys = tmp.map((f) => f.key); + rulesRemove(); + addRules(contractFieldKeys); + }); + }; + + const [currentRuleIndex, setCurrentRuleIndex] = useState(null); + const handleDefaultValueOpen = (rulesIndex: number) => { + setCurrentRuleIndex(rulesIndex); + }; + const handleDefaultValueClose = () => { + setCurrentRuleIndex(null); + }; + + const [openAlertDelete, setOpenAlertDelete] = React.useState(false); + + const handleClickOpenAlertDelete = () => { + setOpenAlertDelete(true); + }; + + const handleCloseAlertDelete = () => { + setOpenAlertDelete(false); + }; + + return ( + <> + + } + > +
+ + #{index + 1} {injectorContractLabel ?? t('New representation')} + + + + + + +
+ + +
+ + + + +
+ + ( + { + onChange(data); + onChangeInjectorContractId(); + }} + error={error} + fieldValue={value} + /> + )} + /> + {rulesFields.map((ruleField, rulesIndex) => { + return ( +
+
+ + {t(ruleField.rule_attribute_name[0].toUpperCase() + ruleField.rule_attribute_name.slice(1))} + {isMandatoryField(ruleField.rule_attribute_name) + && * + } + + ( + + )} + /> + handleDefaultValueOpen(rulesIndex)} + > + + + + +
+ {currentRuleIndex !== null + && + + {t('Attribute mapping configuration')} + + + + {currentRuleIndex === rulesFields.findIndex((r) => r.rule_attribute_name === 'trigger_time') + &&
+ + + + +
+ } + + + + + + } +
+ ); + }) + } + + + + + + + + + + + {t('Do you want to delete this representation?')} + + + + + + + + + ); +}; + +export default RulesContractContent; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterCreation.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterCreation.tsx new file mode 100644 index 0000000000..8db70768a7 --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterCreation.tsx @@ -0,0 +1,60 @@ +import React, { useState } from 'react'; +import { makeStyles } from '@mui/styles'; +import { Fab } from '@mui/material'; +import { Add } from '@mui/icons-material'; +import { useFormatter } from '../../../../../components/i18n'; +import type { ImportMapperAddInput, RawPaginationImportMapper } from '../../../../../utils/api-types'; +import { createMapper } from '../../../../../actions/mapper/mapper-actions'; +import Drawer from '../../../../../components/common/Drawer'; +import MapperForm from './MapperForm'; + +const useStyles = makeStyles(() => ({ + createButton: { + position: 'fixed', + bottom: 30, + right: 230, + }, +})); + +interface Props { + onCreate?: (result: RawPaginationImportMapper) => void; +} + +const XlsFormatterCreation: React.FC = ({ onCreate }) => { + const classes = useStyles(); + const { t } = useFormatter(); + + const [open, setOpen] = useState(false); + + const onSubmit = ((data: ImportMapperAddInput) => { + createMapper(data).then( + (result: { data: RawPaginationImportMapper }) => { + onCreate?.(result.data); + return result; + }, + ); + setOpen(false); + }); + + return ( + <> + setOpen(true)} + color="primary" + aria-label="Add" + className={classes.createButton} + > + + + setOpen(false)} + title={t('Create a xls mapper')} + > + + + + ); +}; + +export default XlsFormatterCreation; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterUpdate.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterUpdate.tsx new file mode 100644 index 0000000000..03a37a10ed --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsFormatterUpdate.tsx @@ -0,0 +1,85 @@ +import React, { FunctionComponent, useEffect, useState } from 'react'; +import MapperForm from './MapperForm'; +import { fetchMapper, updateMapper } from '../../../../../actions/mapper/mapper-actions'; +import type { ImportMapperUpdateInput, RawPaginationImportMapper } from '../../../../../utils/api-types'; +import Loader from '../../../../../components/Loader'; +import { ImportMapperStore } from '../../../../../actions/mapper/mapper'; + +interface XlsFormatterUpdateComponentProps { + xlsMapper: ImportMapperStore; + onUpdate?: (result: RawPaginationImportMapper) => void; + handleClose: () => void; +} + +const XlsFormatterUpdateComponent: FunctionComponent = ({ + xlsMapper, + onUpdate, + handleClose, +}) => { + const initialValues = { + mapper_name: xlsMapper.import_mapper_name ?? '', + mapper_inject_type_column: xlsMapper.import_mapper_inject_type_column ?? '', + mapper_inject_importers: xlsMapper.inject_importers?.map((i) => ({ + inject_importer_injector_contract_id: i.inject_importer_injector_contract, + inject_importer_type_value: i.inject_importer_type_value, + inject_importer_rule_attributes: i.rule_attributes?.map((r) => ({ + rule_attribute_name: r.rule_attribute_name, + rule_attribute_columns: r.rule_attribute_columns, + rule_attribute_default_value: r.rule_attribute_default_value, + rule_attribute_additional_config: r.rule_attribute_additional_config, + })) ?? [], + })) ?? [], + }; + + const onSubmit = ((data: ImportMapperUpdateInput) => { + updateMapper(xlsMapper.import_mapper_id, data).then( + (result: { data: RawPaginationImportMapper }) => { + onUpdate?.(result.data); + return result; + }, + ); + handleClose(); + }); + + return ( + + ); +}; + +interface XlsFormatterUpdateProps { + xlsMapperId: string; + onUpdate?: (result: RawPaginationImportMapper) => void; + handleClose: () => void; +} + +const XlsFormatterUpdate: FunctionComponent = ({ + xlsMapperId, + onUpdate, + handleClose, +}) => { + const [xlsMapper, setXlsMapper] = useState(); + + useEffect(() => { + fetchMapper(xlsMapperId).then((result: { data: ImportMapperStore }) => { + setXlsMapper(result.data); + }); + }, []); + + if (!xlsMapper) { + return ; + } + + return ( + + ); +}; + +export default XlsFormatterUpdate; diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsMapperTestDialog.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsMapperTestDialog.tsx new file mode 100644 index 0000000000..0717cf5a27 --- /dev/null +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_formatter/XlsMapperTestDialog.tsx @@ -0,0 +1,67 @@ +import React, { FunctionComponent, useState } from 'react'; +import { useFormatter } from '../../../../../components/i18n'; +import ImportUploaderInjectFromXlsFile from '../../../common/injects/ImportUploaderInjectFromXlsFile'; +import type { ImportMapperAddInput, ImportPostSummary } from '../../../../../utils/api-types'; +import ImportUploaderInjectFromInjectsTest from '../../../common/injects/ImportUploaderInjectFromInjectsTest'; +import Dialog from '../../../../../components/common/Dialog'; +import { storeXlsFile } from '../../../../../actions/mapper/mapper-actions'; + +interface IngestionCsvMapperTestDialogProps { + open: boolean; + onClose: () => void; + importMapperValues: ImportMapperAddInput; +} + +const XlsMapperTestDialog: FunctionComponent = ({ + open, + onClose, + importMapperValues, +}) => { + // Standard hooks + const { t } = useFormatter(); + + const [importId, setImportId] = useState(null); + const [sheets, setSheets] = useState([]); + + const onSubmitImportFile = (values: { file: File }) => { + storeXlsFile(values.file).then((result: { data: ImportPostSummary }) => { + const { data } = result; + setImportId(data.import_id); + setSheets(data.available_sheets); + }); + }; + + const handleClose = () => { + setImportId(null); + setSheets([]); + onClose(); + }; + + return ( + + <> + {importId === null + && + } + {importId !== null + && + } + + + ); +}; + +export default XlsMapperTestDialog; diff --git a/openbas-front/src/admin/components/simulations/simulation/ExercisePopover.tsx b/openbas-front/src/admin/components/simulations/simulation/ExercisePopover.tsx index 7f05909bb2..a6dd6810f1 100644 --- a/openbas-front/src/admin/components/simulations/simulation/ExercisePopover.tsx +++ b/openbas-front/src/admin/components/simulations/simulation/ExercisePopover.tsx @@ -149,9 +149,10 @@ const ExercisePopover: FunctionComponent = ({ }; const permissions = usePermissions(exercise.exercise_id); + // Button Popover const entries = []; - if (actions.includes('Update')) entries.push({ label: 'Update', action: () => handleOpenEdit() }); - if (actions.includes('Delete')) entries.push({ label: 'Delete', action: () => handleOpenDelete() }); + if (actions.includes('Update')) entries.push({ label: 'Update', action: () => handleOpenEdit(), disabled: !permissions.canWriteBypassStatus }); + if (actions.includes('Delete')) entries.push({ label: 'Delete', action: () => handleOpenDelete(), disabled: !permissions.canWriteBypassStatus }); if (actions.includes('Duplicate')) entries.push({ label: 'Duplicate', action: () => handleOpenDuplicate() }); if (actions.includes('Export')) entries.push({ label: 'Export', action: () => handleOpenExport() }); diff --git a/openbas-front/src/components/InjectContractComponent.tsx b/openbas-front/src/components/InjectContractComponent.tsx new file mode 100644 index 0000000000..b5a8747d45 --- /dev/null +++ b/openbas-front/src/components/InjectContractComponent.tsx @@ -0,0 +1,147 @@ +import React, { FunctionComponent, useEffect, useState } from 'react'; +import { Autocomplete, SelectChangeEvent, TextField } from '@mui/material'; +import { makeStyles } from '@mui/styles'; +import { FieldError } from 'react-hook-form'; +import type { FilterGroup } from '../utils/api-types'; +import { initSorting, Page } from './common/pagination/Page'; +import { useFormatter } from './i18n'; +import InjectIcon from '../admin/components/common/injects/InjectIcon'; +import { isNotEmptyField } from '../utils/utils'; +import { useAppDispatch } from '../utils/hooks'; +import { useHelper } from '../store'; +import type { InjectorHelper } from '../actions/injectors/injector-helper'; +import useDataLoader from '../utils/hooks/useDataLoader'; +import { fetchInjectors } from '../actions/Injectors'; +import { searchInjectorContracts } from '../actions/InjectorContracts'; +import type { InjectorContractStore } from '../actions/injector_contracts/InjectorContract'; + +const useStyles = makeStyles(() => ({ + icon: { + paddingTop: 4, + display: 'inline-block', + }, + text: { + display: 'inline-block', + flexGrow: 1, + marginLeft: 10, + }, +})); + +interface Props { + label: string; + onChange: (data: string | null | undefined) => void; + error: FieldError | undefined; + fieldValue: string | undefined; +} + +const InjectContractComponent: FunctionComponent = ({ + label, + onChange, + error, + fieldValue, +}) => { + // Standard hooks + const classes = useStyles(); + const { t, tPick } = useFormatter(); + const dispatch = useAppDispatch(); + + // Fetching data + const { injectorMap } = useHelper((helper: InjectorHelper) => ({ + injectorMap: helper.getInjectorsMap(), + })); + useDataLoader(() => { + dispatch(fetchInjectors()); + }); + + // Pagination + const [contracts, setContracts] = useState([]); + const searchContract = (event: React.SyntheticEvent) => { + const selectChangeEvent = event as SelectChangeEvent; + const val = selectChangeEvent?.target.value ?? ''; + return contracts.filter( + (type) => type.injector_contract_id.includes(val) + || tPick(type.injector_contract_labels).includes(val), + ); + }; + + const importFilter: FilterGroup = { + mode: 'and', + filters: [ + { + key: 'injector_contract_import_available', + operator: 'eq', + mode: 'and', + values: ['true'], + }], + }; + const searchPaginationInput = { + sorts: initSorting('injector_contract_labels'), + filterGroup: importFilter, + }; + + useEffect(() => { + const finalSearchPaginationInput = { + ...searchPaginationInput, + textSearch: '', + page: 0, + size: 100, + }; + + searchInjectorContracts(finalSearchPaginationInput).then((result: { data: Page }) => { + const { data } = result; + setContracts(data.content); + }); + }, []); + + const [value, setValue] = React.useState(fieldValue ?? ''); + + return ( + tPick(option.injector_contract_labels)} + renderInput={ + (params) => ( + + ) + } + options={contracts} + value={contracts.find((i) => i.injector_contract_id === value) ?? null} + onChange={(_event, injectorContract) => { + setValue(injectorContract?.injector_contract_id); + onChange(injectorContract?.injector_contract_id); + }} + onInputChange={(event) => searchContract(event)} + renderOption={(props, option) => ( +
  • +
    + +
    +
    + {tPick(option.injector_contract_labels)} +
    +
    • + )} + /> + ); +}; + +export default InjectContractComponent; diff --git a/openbas-front/src/components/RegexComponent.tsx b/openbas-front/src/components/RegexComponent.tsx new file mode 100644 index 0000000000..62bcd32d3f --- /dev/null +++ b/openbas-front/src/components/RegexComponent.tsx @@ -0,0 +1,59 @@ +import React from 'react'; +import { Autocomplete, TextField } from '@mui/material'; +import { FieldError } from 'react-hook-form'; +import alphabet from '../admin/components/settings/data_ingestion/AttributeUtils'; +import { useFormatter } from './i18n'; + +interface Props { + label: string; + fieldValue: string | null | undefined; + onChange: (data: string | null) => void; + required?: boolean; + error: FieldError | undefined; +} + +const RegexComponent: React.FC = ({ + label, + fieldValue, + onChange, + required, + error, +}) => { + // Standard hooks + const { t } = useFormatter(); + + const regexOptions = alphabet(26); + const [value, setValue] = React.useState(fieldValue ?? ''); + + const inputLabelProps = required ? { required: true } : {}; + + return ( + ( + + ) + } + options={regexOptions} + value={regexOptions.find((r) => r === value) ?? null} + onChange={(_event, newValue) => { + setValue(newValue); + onChange(newValue); + }} + /> + ); +}; + +export default RegexComponent; diff --git a/openbas-front/src/components/common/ButtonPopover.tsx b/openbas-front/src/components/common/ButtonPopover.tsx index 7f3a5e01c5..f53d661e73 100644 --- a/openbas-front/src/components/common/ButtonPopover.tsx +++ b/openbas-front/src/components/common/ButtonPopover.tsx @@ -3,7 +3,7 @@ import { MoreVert } from '@mui/icons-material'; import React, { FunctionComponent, useState } from 'react'; import { useFormatter } from '../i18n'; -export interface ButtonPopoverEntry { +export interface PopoverEntry { label: string; action: () => void | React.Dispatch>; disabled?: boolean; @@ -12,7 +12,7 @@ export interface ButtonPopoverEntry { export type VariantButtonPopover = 'toggle' | 'icon'; interface Props { - entries: ButtonPopoverEntry[]; + entries: PopoverEntry[]; buttonProps?: ToggleButtonProps; variant?: VariantButtonPopover; } diff --git a/openbas-front/src/components/common/CodeBlock.tsx b/openbas-front/src/components/common/CodeBlock.tsx new file mode 100644 index 0000000000..b644cfcf46 --- /dev/null +++ b/openbas-front/src/components/common/CodeBlock.tsx @@ -0,0 +1,26 @@ +import React, { FunctionComponent } from 'react'; +import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; +import { a11yDark, coy } from 'react-syntax-highlighter/dist/esm/styles/prism'; +import { useTheme } from '@mui/styles'; +import { Theme } from '@mui/material'; + +interface CodeBlockProps { + code: string; + language: string; +} + +const CodeBlock: FunctionComponent = ({ language, code }) => { + const theme = useTheme(); + return ( + + {code} + + ); +}; + +export default CodeBlock; diff --git a/openbas-front/src/components/common/CustomFileUploader.tsx b/openbas-front/src/components/common/CustomFileUploader.tsx new file mode 100644 index 0000000000..e22c131bf8 --- /dev/null +++ b/openbas-front/src/components/common/CustomFileUploader.tsx @@ -0,0 +1,145 @@ +import React, { FormEvent, FunctionComponent, useEffect, useState } from 'react'; +import { Box, Button, InputLabel } from '@mui/material'; +import { Theme } from '@mui/material/styles/createTheme'; +import { makeStyles } from '@mui/styles'; +import classNames from 'classnames'; +import { FieldError, FieldErrors, FieldErrorsImpl, Merge } from 'react-hook-form'; +import VisuallyHiddenInput from './VisuallyHiddenInput'; +import { useFormatter } from '../i18n'; +import { truncate } from '../../utils/String'; + +interface CustomFileUploadProps { + name: string; + fieldOnChange: (value: File | string | undefined) => void; + label?: string; + errors: FieldErrors; + acceptMimeTypes?: string; // html input "accept" with MIME types only + sizeLimit?: number; // in bytes +} + +const useStyles = makeStyles((theme) => ({ + box: { + width: '100%', + marginTop: '0.2rem', + paddingBottom: '0.35rem', + borderBottom: `0.1rem solid ${theme.palette.grey['400']}`, + cursor: 'default', + '&:hover': { + borderBottom: '0.1rem solid white', + }, + '&:active': { + borderBottom: `0.1rem solid ${theme.palette.primary.main}`, + }, + }, + boxError: { + borderBottom: `0.1rem solid ${theme.palette.error.main}`, + }, + button: { + lineHeight: '0.65rem', + }, + div: { + marginTop: 20, + width: '100%', + }, + error: { + color: theme.palette.error.main, + fontSize: '0.75rem', + }, + span: { + marginLeft: 5, + verticalAlign: 'bottom', + }, +})); + +const CustomFileUploader: FunctionComponent = ({ + name, + fieldOnChange, + label, + acceptMimeTypes, + sizeLimit = 0, // defaults to 0 = no limit + errors, +}) => { + const { t } = useFormatter(); + const classes = useStyles(); + const [fileNameForDisplay, setFileNameForDisplay] = useState(''); + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const [errorText, setErrorText] = useState>>(''); + + useEffect(() => { + if (errors[name]) { + setErrorText('Should be a valid XLS file'); + } else { + setErrorText(''); + } + }, [errors[name]]); + + const onChange = async (event: FormEvent) => { + const inputElement = event.target as HTMLInputElement; + const eventTargetValue = inputElement.value as string; + const file = inputElement.files?.[0]; + const fileSize = file?.size || 0; + + const newFileName = eventTargetValue.substring( + eventTargetValue.lastIndexOf('\\') + 1, + ); + setFileNameForDisplay(truncate(newFileName, 60)); + setErrorText(''); + + // check the file type; user might still provide something bypassing 'accept' + // this will work only if accept is using MIME types only + const acceptedList = acceptMimeTypes?.split(',').map((a) => a.trim()) || []; + if ( + acceptedList.length > 0 + && !!file?.type + && !acceptedList.includes(file?.type) + ) { + setErrorText(t('This file is not in the specified format')); + return; + } + + // check the size limit if any set; if file is too big it is not set as value + if (fileSize > 0 && sizeLimit > 0 && fileSize > sizeLimit) { + setErrorText(t('This file is too large')); + return; + } + + fieldOnChange(inputElement.files?.[0]); + }; + + return ( +
    + + {label ? t(label) : t('Associated file')} + + + + + {fileNameForDisplay || t('No file selected.')} + + + {!!errorText && ( +
    + {t(errorText)} +
    + )} +
    + ); +}; + +export default CustomFileUploader; diff --git a/openbas-front/src/components/common/IconPopover.tsx b/openbas-front/src/components/common/IconPopover.tsx new file mode 100644 index 0000000000..e0765b8a8c --- /dev/null +++ b/openbas-front/src/components/common/IconPopover.tsx @@ -0,0 +1,56 @@ +import { IconButton, Menu, MenuItem } from '@mui/material'; +import { MoreVert } from '@mui/icons-material'; +import React, { FunctionComponent, useState } from 'react'; +import { useFormatter } from '../i18n'; +import { PopoverEntry } from './ButtonPopover'; + +interface Props { + entries: PopoverEntry[]; +} + +const IconPopover: FunctionComponent = ({ + entries, +}) => { + // Standard hooks + const { t } = useFormatter(); + + const [anchorEl, setAnchorEl] = useState(null); + + return ( + <> + { + ev.stopPropagation(); + setAnchorEl(ev.currentTarget); + }} + aria-label={'Xls formatter menu'} + aria-haspopup="true" + size="large" + > + + + setAnchorEl(null)} + > + {entries.map((entry) => { + return ( + { + entry.action(); + setAnchorEl(null); + }} + > + {t(entry.label)} + + ); + })} + + + ); +}; + +export default IconPopover; diff --git a/openbas-front/src/components/common/VisuallyHiddenInput.tsx b/openbas-front/src/components/common/VisuallyHiddenInput.tsx new file mode 100644 index 0000000000..ebd143d033 --- /dev/null +++ b/openbas-front/src/components/common/VisuallyHiddenInput.tsx @@ -0,0 +1,15 @@ +import { styled } from '@mui/material/styles'; + +const VisuallyHiddenInput = styled('input')` + clip: rect(0 0 0 0); + clip-path: inset(50%); + height: 1rem; + overflow: hidden; + position: absolute; + bottom: 0; + left: 0; + white-space: nowrap; + width: 1rem; +`; + +export default VisuallyHiddenInput; diff --git a/openbas-front/src/components/fields/FileLoader.tsx b/openbas-front/src/components/fields/FileLoader.tsx index 9149e2a13b..71ad8a06ca 100644 --- a/openbas-front/src/components/fields/FileLoader.tsx +++ b/openbas-front/src/components/fields/FileLoader.tsx @@ -12,7 +12,7 @@ import ItemTags from '../ItemTags'; import type { Theme } from '../Theme'; import { useFormatter } from '../i18n'; import DocumentType from '../../admin/components/components/documents/DocumentType'; -import ButtonPopover, { ButtonPopoverEntry } from '../common/ButtonPopover'; +import ButtonPopover, { PopoverEntry } from '../common/ButtonPopover'; import FileTransferDialog from './FileTransferDialog'; const useStyles = makeStyles((theme: Theme) => ({ @@ -154,7 +154,7 @@ const FileLoader: React.FC = ({ }; // Button Popover entries - const entries: ButtonPopoverEntry[] = [ + const entries: PopoverEntry[] = [ { label: 'Update', action: handleOpen }, { label: 'Remove', action: handleRemove }, { label: 'Download', action: () => handleDownload(selectedDocument?.document_id) }, diff --git a/openbas-front/src/utils/Action.js b/openbas-front/src/utils/Action.js index 6d0a65faae..18807d6f15 100644 --- a/openbas-front/src/utils/Action.js +++ b/openbas-front/src/utils/Action.js @@ -31,9 +31,25 @@ const buildError = (data) => { }; export const simpleCall = (uri) => api().get(buildUri(uri)); -export const simplePostCall = (uri, data) => api().post(buildUri(uri), data); -export const simplePutCall = (uri, data) => api().put(buildUri(uri), data); -export const simpleDelCall = (uri, data) => api().delete(buildUri(uri), data); +export const simplePostCall = (uri, data) => api().post(buildUri(uri), data) + .catch((error) => { + MESSAGING$.notifyError(error.message); + throw error; + }); +export const simplePutCall = (uri, data) => api().put(buildUri(uri), data) + .then((response) => { + MESSAGING$.notifySuccess('The element has been updated'); + return response; + }) + .catch((error) => { + MESSAGING$.notifyError(error.message); + throw error; + }); +export const simpleDelCall = (uri, data) => api().delete(buildUri(uri), data) + .catch((error) => { + MESSAGING$.notifyError(error.message); + throw error; + }); export const getReferential = (schema, uri, noloading) => (dispatch) => { if (noloading !== true) { dispatch({ type: Constants.DATA_FETCH_SUBMITTED }); diff --git a/openbas-front/src/utils/Localization.js b/openbas-front/src/utils/Localization.js index 6f9b605d92..486ba58b1d 100644 --- a/openbas-front/src/utils/Localization.js +++ b/openbas-front/src/utils/Localization.js @@ -525,7 +525,7 @@ const i18n = { 'Primary color (light)': 'Couleur primaire (clair)', 'Secondary color (light)': 'Couleur secondaire (clair)', logo: 'Logo seulement', - title: 'Titre seulement', + 'Title only': 'Titre seulement', 'logo-title': 'Logo et titre', 'Dark theme': 'Thème sombre', 'Light theme': 'Thème clair', @@ -1057,6 +1057,7 @@ const i18n = { backlabel: 'étiquette de retour', backuri: 'uri de retour', 'Injects Results': 'Résultats des stimulis', + 'Import injects': 'Importer des stimulis', 'Browse the link': 'Parcourir le lien', EE: 'EE', 'Error or prevented': 'Erreur ou empêché', @@ -1116,18 +1117,46 @@ const i18n = { 'Cleanup executor': 'Exécuteur de nettoyage', 'Cleanup command': 'Commande de nettoyage', Document: 'Document', - // -- Policies + // Policies 'Platform login message': 'Messages de connexion', 'Platform consent message': 'Message de consentement à la plate-forme', 'Platform consent confirm text': 'Texte de confirmation du consentement à la plate-forme', Write: 'Ecriture', - // -- Timeline + // Timeline 'Hide timeline': 'Cacher la chronologie', 'Show Timeline': 'Afficher la chronologie', COMMUNITY: 'Community', FILIGRAN: 'Filigran', VERIFIED: 'Verified', UNVERIFIED: 'Unverified', + // ---Xls mapper + 'Create a xls mapper': 'Créer un mapper xls', + 'Mapper name': 'Nom du mapper', + 'Inject type column': 'Colonne de type d\'inject', + 'Representation for inject type': 'Représentation de l\'inject type', + 'Inject importer name': 'Nom d\'inject importer', + 'Matching type in the xls': 'Type correspondant dans le xls', + 'Inject importer description': 'Description d\'inject importer', + 'Inject type': 'Typed d\'inject', + 'Inject importer rules': 'Règles d\'inject importer', + 'At least one inject type is required': 'Au moins un inject type est requis', + 'New representation': 'Nouvelle représentation', + 'Rule attribute name': 'Nom de règle d\'attribut', + 'Rule attribute columns': 'Colonnes de règle d\'attribut', + 'Time pattern': 'Modèle de temps', + 'Rule title': 'Titre de la règle', + 'Rule description': 'Description de la règle', + 'Do you want to delete this representation?': 'Voulez-vous supprimer cette représentation ?', + 'Trigger time': 'Heure de lancement', + 'Attribute mapping configuration': 'Configuration d\'attribut de mappeur', + Test: 'Test', + 'Do you want to delete this XLS mapper ?': 'Voulez-vous supprimer ce mappage XLS ?', + Expectation_name: 'Nom de l\'attendu', + Expectation_description: 'Description de l\'attendu', + Expectation_score: 'Score de l\'attendu', + Trigger_time: 'Temps de déclenchement', + 'This expression will be used to find a match in the specified column to determine the inject type. Regular Expressions can be used.': 'Cette expression est utilisé pour déterminer si il y a correspondance avec la colonne spécifié. L\'utilisation d\'expression régulière est possible.', + 'By default we accept iso date (YYYY-MM-DDThh:mm:ss[.mmm]TZD), but you can specify your own date format in ISO notation (for instance DD.MM.YYYY hh\'h\'mm)': 'Par défaut nous acceptons la date iso (AAAA-MM-JJ hh:mm:ss[.mmm]TZD), mais vous pouvez spécifiez votre propre format de date en notation ISO (par exemple JJ.MM.AAAA hh\'h\'mm)', }, en: { openbas_email: 'Email', @@ -1158,7 +1187,6 @@ const i18n = { tv: 'TV channel', microblogging: 'Microblogging', logo: 'Logo only', - title: 'Title only', 'logo-title': 'Logo and title', VALUE: 'Text', VALUE_CASE: 'Text (case-sensitive)', @@ -1213,6 +1241,11 @@ const i18n = { FILIGRAN: 'Filigran', VERIFIED: 'Verified', UNVERIFIED: 'Unverified', + // - XLS Mapper + Expectation_name: 'Expectation name', + Expectation_description: 'Expectation description', + Expectation_score: 'Expectation score', + Trigger_time: 'Trigger time', }, }, }; diff --git a/openbas-front/src/utils/api-types.d.ts b/openbas-front/src/utils/api-types.d.ts index aa82838e0c..9f391c0572 100644 --- a/openbas-front/src/utils/api-types.d.ts +++ b/openbas-front/src/utils/api-types.d.ts @@ -975,6 +975,54 @@ export interface GroupUpdateUsersInput { group_users?: string[]; } +export interface ImportMapper { + /** @format date-time */ + import_mapper_created_at?: string; + import_mapper_id: string; + import_mapper_inject_type_column: string; + import_mapper_name: string; + /** @format date-time */ + import_mapper_updated_at?: string; + inject_importers?: InjectImporter[]; + updateAttributes?: object; +} + +export interface ImportMapperAddInput { + mapper_inject_importers: InjectImporterAddInput[]; + /** @pattern ^[A-Z]{1,2}$ */ + mapper_inject_type_column: string; + mapper_name: string; +} + +export interface ImportMapperUpdateInput { + mapper_inject_importers: InjectImporterUpdateInput[]; + /** @pattern ^[A-Z]{1,2}$ */ + mapper_inject_type_column: string; + mapper_name: string; +} + +export interface ImportMessage { + message_code?: + | "NO_POTENTIAL_MATCH_FOUND" + | "SEVERAL_MATCHES" + | "ABSOLUTE_TIME_WITHOUT_START_DATE" + | "DATE_SET_IN_PAST" + | "DATE_SET_IN_FUTURE" + | "NO_TEAM_FOUND"; + message_level?: "CRITICAL" | "ERROR" | "WARN" | "INFO"; + message_params?: Record; +} + +export interface ImportPostSummary { + available_sheets: string[]; + import_id: string; +} + +export interface ImportTestSummary { + import_message?: ImportMessage[]; + injects?: InjectResultDTO[]; +} + export interface Inject { footer?: string; header?: string; @@ -1108,6 +1156,31 @@ export interface InjectExpectationUpdateInput { success?: boolean; } +export interface InjectImporter { + /** @format date-time */ + inject_importer_created_at?: string; + inject_importer_id: string; + inject_importer_injector_contract: InjectorContract; + inject_importer_type_value: string; + /** @format date-time */ + inject_importer_updated_at?: string; + rule_attributes?: RuleAttribute[]; + updateAttributes?: object; +} + +export interface InjectImporterAddInput { + inject_importer_injector_contract_id: string; + inject_importer_rule_attributes?: RuleAttributeAddInput[]; + inject_importer_type_value: string; +} + +export interface InjectImporterUpdateInput { + inject_importer_id?: string; + inject_importer_injector_contract_id: string; + inject_importer_rule_attributes?: RuleAttributeUpdateInput[]; + inject_importer_type_value: string; +} + export interface InjectInput { inject_all_teams?: boolean; inject_asset_groups?: string[]; @@ -1299,7 +1372,8 @@ export interface InjectorContract { injector_contract_created_at?: string; injector_contract_custom?: boolean; injector_contract_id: string; - injector_contract_injector?: Injector; + injector_contract_import_available?: boolean; + injector_contract_injector: Injector; injector_contract_injector_type?: string; injector_contract_labels?: Record; injector_contract_manual?: boolean; @@ -1386,6 +1460,20 @@ export interface InjectorUpdateInput { injector_payloads?: boolean; } +export interface InjectsImportInput { + import_mapper_id: string; + sheet_name: string; + /** @format int32 */ + timezone_offset: number; +} + +export interface InjectsImportTestInput { + import_mapper: ImportMapperAddInput; + sheet_name: string; + /** @format int32 */ + timezone_offset: number; +} + export type JsonNode = object; export interface KillChainPhase { @@ -1972,6 +2060,25 @@ export interface PageRawPaginationDocument { totalPages?: number; } +export interface PageRawPaginationImportMapper { + content?: RawPaginationImportMapper[]; + empty?: boolean; + first?: boolean; + last?: boolean; + /** @format int32 */ + number?: number; + /** @format int32 */ + numberOfElements?: number; + pageable?: PageableObject; + /** @format int32 */ + size?: number; + sort?: SortObject[]; + /** @format int64 */ + totalElements?: number; + /** @format int32 */ + totalPages?: number; +} + export interface PageRawPaginationPlayer { content?: RawPaginationPlayer[]; empty?: boolean; @@ -2353,6 +2460,15 @@ export interface RawPaginationDocument { document_type?: string; } +export interface RawPaginationImportMapper { + /** @format date-time */ + import_mapper_created_at?: string; + import_mapper_id: string; + import_mapper_name?: string; + /** @format date-time */ + import_mapper_updated_at?: string; +} + export interface RawPaginationPlayer { user_email?: string; user_firstname?: string; @@ -2465,6 +2581,34 @@ export interface ResultDistribution { value: number; } +export interface RuleAttribute { + rule_attribute_additional_config?: Record; + rule_attribute_columns?: string; + /** @format date-time */ + rule_attribute_created_at?: string; + rule_attribute_default_value?: string; + rule_attribute_id: string; + rule_attribute_name: string; + /** @format date-time */ + rule_attribute_updated_at?: string; + updateAttributes?: object; +} + +export interface RuleAttributeAddInput { + rule_attribute_additional_config?: Record; + rule_attribute_columns?: string | null; + rule_attribute_default_value?: string; + rule_attribute_name: string; +} + +export interface RuleAttributeUpdateInput { + rule_attribute_additional_config?: Record; + rule_attribute_columns?: string | null; + rule_attribute_default_value?: string; + rule_attribute_id?: string; + rule_attribute_name: string; +} + export interface Scenario { /** @format int64 */ scenario_all_users_number?: number; diff --git a/openbas-front/vite.config.ts b/openbas-front/vite.config.ts index 1ebab9b8fd..dc3268de9a 100644 --- a/openbas-front/vite.config.ts +++ b/openbas-front/vite.config.ts @@ -74,6 +74,7 @@ export default ({ mode }: { mode: string }) => { 'reactflow', 'd3-hierarchy', '@dagrejs/dagre', + 'react-syntax-highlighter', ], }, diff --git a/openbas-front/yarn.lock b/openbas-front/yarn.lock index a7eb9ed46a..ade39a90e8 100644 --- a/openbas-front/yarn.lock +++ b/openbas-front/yarn.lock @@ -4563,6 +4563,15 @@ __metadata: languageName: node linkType: hard +"@types/react-syntax-highlighter@npm:15": + version: 15.5.13 + resolution: "@types/react-syntax-highlighter@npm:15.5.13" + dependencies: + "@types/react": "npm:*" + checksum: 10c0/e3bca325b27519fb063d3370de20d311c188ec16ffc01e5bc77bdf2d7320756725ee3d0246922cd5d38b75c5065a1bc43d0194e92ecf6556818714b4ffb0967a + languageName: node + linkType: hard + "@types/react-transition-group@npm:^4.4.10, @types/react-transition-group@npm:^4.4.8": version: 4.4.10 resolution: "@types/react-transition-group@npm:4.4.10" @@ -6131,6 +6140,13 @@ __metadata: languageName: node linkType: hard +"character-entities-legacy@npm:^1.0.0": + version: 1.1.4 + resolution: "character-entities-legacy@npm:1.1.4" + checksum: 10c0/ea4ca9c29887335eed86d78fc67a640168342b1274da84c097abb0575a253d1265281a5052f9a863979e952bcc267b4ecaaf4fe233a7e1e0d8a47806c65b96c7 + languageName: node + linkType: hard + "character-entities-legacy@npm:^3.0.0": version: 3.0.0 resolution: "character-entities-legacy@npm:3.0.0" @@ -6138,6 +6154,13 @@ __metadata: languageName: node linkType: hard +"character-entities@npm:^1.0.0": + version: 1.2.4 + resolution: "character-entities@npm:1.2.4" + checksum: 10c0/ad015c3d7163563b8a0ee1f587fb0ef305ef344e9fd937f79ca51cccc233786a01d591d989d5bf7b2e66b528ac9efba47f3b1897358324e69932f6d4b25adfe1 + languageName: node + linkType: hard + "character-entities@npm:^2.0.0": version: 2.0.2 resolution: "character-entities@npm:2.0.2" @@ -6145,6 +6168,13 @@ __metadata: languageName: node linkType: hard +"character-reference-invalid@npm:^1.0.0": + version: 1.1.4 + resolution: "character-reference-invalid@npm:1.1.4" + checksum: 10c0/29f05081c5817bd1e975b0bf61e77b60a40f62ad371d0f0ce0fdb48ab922278bc744d1fbe33771dced751887a8403f265ff634542675c8d7375f6ff4811efd0e + languageName: node + linkType: hard + "character-reference-invalid@npm:^2.0.0": version: 2.0.1 resolution: "character-reference-invalid@npm:2.0.1" @@ -6421,6 +6451,13 @@ __metadata: languageName: node linkType: hard +"comma-separated-tokens@npm:^1.0.0": + version: 1.0.8 + resolution: "comma-separated-tokens@npm:1.0.8" + checksum: 10c0/c3bcfeaa6d50313528a006a40bcc0f9576086665c9b48d4b3a76ddd63e7d6174734386c98be1881cbf6ecfc25e1db61cd775a7b896d2ea7a65de28f83a0f9b17 + languageName: node + linkType: hard + "comma-separated-tokens@npm:^2.0.0": version: 2.0.3 resolution: "comma-separated-tokens@npm:2.0.3" @@ -8378,6 +8415,15 @@ __metadata: languageName: node linkType: hard +"fault@npm:^1.0.0": + version: 1.0.4 + resolution: "fault@npm:1.0.4" + dependencies: + format: "npm:^0.2.0" + checksum: 10c0/c86c11500c1b676787296f31ade8473adcc6784f118f07c1a9429730b6288d0412f96e069ce010aa57e4f65a9cccb5abee8868bbe3c5f10de63b20482c9baebd + languageName: node + linkType: hard + "file-entry-cache@npm:^6.0.1": version: 6.0.1 resolution: "file-entry-cache@npm:6.0.1" @@ -8553,6 +8599,13 @@ __metadata: languageName: node linkType: hard +"format@npm:^0.2.0": + version: 0.2.2 + resolution: "format@npm:0.2.2" + checksum: 10c0/6032ba747541a43abf3e37b402b2f72ee08ebcb58bf84d816443dd228959837f1cddf1e8775b29fa27ff133f4bd146d041bfca5f9cf27f048edf3d493cf8fee6 + languageName: node + linkType: hard + "forwarded@npm:0.2.0": version: 0.2.0 resolution: "forwarded@npm:0.2.0" @@ -9033,6 +9086,13 @@ __metadata: languageName: node linkType: hard +"hast-util-parse-selector@npm:^2.0.0": + version: 2.2.5 + resolution: "hast-util-parse-selector@npm:2.2.5" + checksum: 10c0/29b7ee77960ded6a99d30c287d922243071cc07b39f2006f203bd08ee54eb8f66bdaa86ef6527477c766e2382d520b60ee4e4087f189888c35d8bcc020173648 + languageName: node + linkType: hard + "hast-util-parse-selector@npm:^3.0.0": version: 3.1.1 resolution: "hast-util-parse-selector@npm:3.1.1" @@ -9172,6 +9232,19 @@ __metadata: languageName: node linkType: hard +"hastscript@npm:^6.0.0": + version: 6.0.0 + resolution: "hastscript@npm:6.0.0" + dependencies: + "@types/hast": "npm:^2.0.0" + comma-separated-tokens: "npm:^1.0.0" + hast-util-parse-selector: "npm:^2.0.0" + property-information: "npm:^5.0.0" + space-separated-tokens: "npm:^1.0.0" + checksum: 10c0/f76d9cf373cb075c8523c8ad52709f09f7e02b7c9d3152b8d35c65c265b9f1878bed6023f215a7d16523921036d40a7da292cb6f4399af9b5eccac2a5a5eb330 + languageName: node + linkType: hard + "hastscript@npm:^7.0.0": version: 7.2.0 resolution: "hastscript@npm:7.2.0" @@ -9198,6 +9271,13 @@ __metadata: languageName: node linkType: hard +"highlight.js@npm:^10.4.1, highlight.js@npm:~10.7.0": + version: 10.7.3 + resolution: "highlight.js@npm:10.7.3" + checksum: 10c0/073837eaf816922427a9005c56c42ad8786473dc042332dfe7901aa065e92bc3d94ebf704975257526482066abb2c8677cc0326559bb8621e046c21c5991c434 + languageName: node + linkType: hard + "history@npm:5.3.0": version: 5.3.0 resolution: "history@npm:5.3.0" @@ -9552,6 +9632,13 @@ __metadata: languageName: node linkType: hard +"is-alphabetical@npm:^1.0.0": + version: 1.0.4 + resolution: "is-alphabetical@npm:1.0.4" + checksum: 10c0/1505b1de5a1fd74022c05fb21b0e683a8f5229366bac8dc4d34cf6935bcfd104d1125a5e6b083fb778847629f76e5bdac538de5367bdf2b927a1356164e23985 + languageName: node + linkType: hard + "is-alphabetical@npm:^2.0.0": version: 2.0.1 resolution: "is-alphabetical@npm:2.0.1" @@ -9559,6 +9646,16 @@ __metadata: languageName: node linkType: hard +"is-alphanumerical@npm:^1.0.0": + version: 1.0.4 + resolution: "is-alphanumerical@npm:1.0.4" + dependencies: + is-alphabetical: "npm:^1.0.0" + is-decimal: "npm:^1.0.0" + checksum: 10c0/d623abae7130a7015c6bf33d99151d4e7005572fd170b86568ff4de5ae86ac7096608b87dd4a1d4dbbd497e392b6396930ba76c9297a69455909cebb68005905 + languageName: node + linkType: hard + "is-alphanumerical@npm:^2.0.0": version: 2.0.1 resolution: "is-alphanumerical@npm:2.0.1" @@ -9667,6 +9764,13 @@ __metadata: languageName: node linkType: hard +"is-decimal@npm:^1.0.0": + version: 1.0.4 + resolution: "is-decimal@npm:1.0.4" + checksum: 10c0/a4ad53c4c5c4f5a12214e7053b10326711f6a71f0c63ba1314a77bd71df566b778e4ebd29f9fb6815f07a4dc50c3767fb19bd6fc9fa05e601410f1d64ffeac48 + languageName: node + linkType: hard + "is-decimal@npm:^2.0.0": version: 2.0.1 resolution: "is-decimal@npm:2.0.1" @@ -9715,6 +9819,13 @@ __metadata: languageName: node linkType: hard +"is-hexadecimal@npm:^1.0.0": + version: 1.0.4 + resolution: "is-hexadecimal@npm:1.0.4" + checksum: 10c0/ec4c64e5624c0f240922324bc697e166554f09d3ddc7633fc526084502626445d0a871fbd8cae52a9844e83bd0bb414193cc5a66806d7b2867907003fc70c5ea + languageName: node + linkType: hard + "is-hexadecimal@npm:^2.0.0": version: 2.0.1 resolution: "is-hexadecimal@npm:2.0.1" @@ -10567,6 +10678,16 @@ __metadata: languageName: node linkType: hard +"lowlight@npm:^1.17.0": + version: 1.20.0 + resolution: "lowlight@npm:1.20.0" + dependencies: + fault: "npm:^1.0.0" + highlight.js: "npm:~10.7.0" + checksum: 10c0/728bce6f6fe8b157f48d3324e597f452ce0eed2ccff1c0f41a9047380f944e971eb45bceb31f08fbb64d8f338dabb166f10049b35b92c7ec5cf0241d6adb3dea + languageName: node + linkType: hard + "lru-cache@npm:^10.0.1, lru-cache@npm:^9.1.1 || ^10.0.0": version: 10.2.0 resolution: "lru-cache@npm:10.2.0" @@ -11934,6 +12055,7 @@ __metadata: "@types/react": "npm:18.3.3" "@types/react-csv": "npm:1.1.10" "@types/react-dom": "npm:18.3.0" + "@types/react-syntax-highlighter": "npm:15" "@types/seamless-immutable": "npm:7.1.19" "@types/uuid": "npm:9.0.8" "@typescript-eslint/eslint-plugin": "npm:7.10.0" @@ -11998,6 +12120,7 @@ __metadata: react-markdown: "npm:9.0.1" react-redux: "npm:8.1.3" react-router-dom: "npm:6.23.1" + react-syntax-highlighter: "npm:15.5.0" reactflow: "npm:11.11.3" redux: "npm:4.2.1" redux-first-history: "npm:5.2.0" @@ -12130,6 +12253,20 @@ __metadata: languageName: node linkType: hard +"parse-entities@npm:^2.0.0": + version: 2.0.0 + resolution: "parse-entities@npm:2.0.0" + dependencies: + character-entities: "npm:^1.0.0" + character-entities-legacy: "npm:^1.0.0" + character-reference-invalid: "npm:^1.0.0" + is-alphanumerical: "npm:^1.0.0" + is-decimal: "npm:^1.0.0" + is-hexadecimal: "npm:^1.0.0" + checksum: 10c0/f85a22c0ea406ff26b53fdc28641f01cc36fa49eb2e3135f02693286c89ef0bcefc2262d99b3688e20aac2a14fd10b75c518583e875c1b9fe3d1f937795e0854 + languageName: node + linkType: hard + "parse-entities@npm:^4.0.0": version: 4.0.1 resolution: "parse-entities@npm:4.0.1" @@ -12849,6 +12986,20 @@ __metadata: languageName: node linkType: hard +"prismjs@npm:^1.27.0": + version: 1.29.0 + resolution: "prismjs@npm:1.29.0" + checksum: 10c0/d906c4c4d01b446db549b4f57f72d5d7e6ccaca04ecc670fb85cea4d4b1acc1283e945a9cbc3d81819084a699b382f970e02f9d1378e14af9808d366d9ed7ec6 + languageName: node + linkType: hard + +"prismjs@npm:~1.27.0": + version: 1.27.0 + resolution: "prismjs@npm:1.27.0" + checksum: 10c0/841cbf53e837a42df9155c5ce1be52c4a0a8967ac916b52a27d066181a3578186c634e52d06d0547fb62b65c486b99b95f826dd54966619f9721b884f486b498 + languageName: node + linkType: hard + "proc-log@npm:^3.0.0": version: 3.0.0 resolution: "proc-log@npm:3.0.0" @@ -12893,6 +13044,15 @@ __metadata: languageName: node linkType: hard +"property-information@npm:^5.0.0": + version: 5.6.0 + resolution: "property-information@npm:5.6.0" + dependencies: + xtend: "npm:^4.0.0" + checksum: 10c0/d54b77c31dc13bb6819559080b2c67d37d94be7dc271f404f139a16a57aa96fcc0b3ad806d4a5baef9e031744853e4afe3df2e37275aacb1f78079bbb652c5af + languageName: node + linkType: hard + "property-information@npm:^6.0.0": version: 6.4.1 resolution: "property-information@npm:6.4.1" @@ -13258,6 +13418,21 @@ __metadata: languageName: node linkType: hard +"react-syntax-highlighter@npm:15.5.0": + version: 15.5.0 + resolution: "react-syntax-highlighter@npm:15.5.0" + dependencies: + "@babel/runtime": "npm:^7.3.1" + highlight.js: "npm:^10.4.1" + lowlight: "npm:^1.17.0" + prismjs: "npm:^1.27.0" + refractor: "npm:^3.6.0" + peerDependencies: + react: ">= 0.14.0" + checksum: 10c0/2bf57a1ea151f688efc7eba355677577c9bb55f05f9df7ef86627aae42f63f505486cddf3f4a628aecc51ec75e89beb9533201570d03201c4bf7d69d61d2545d + languageName: node + linkType: hard + "react-transition-group@npm:4.4.5": version: 4.4.5 resolution: "react-transition-group@npm:4.4.5" @@ -13398,6 +13573,17 @@ __metadata: languageName: node linkType: hard +"refractor@npm:^3.6.0": + version: 3.6.0 + resolution: "refractor@npm:3.6.0" + dependencies: + hastscript: "npm:^6.0.0" + parse-entities: "npm:^2.0.0" + prismjs: "npm:~1.27.0" + checksum: 10c0/63ab62393c8c2fd7108c2ea1eff721c0ad2a1a6eee60fdd1b47f4bb25cf298667dc97d041405b3e718b0817da12b37a86ed07ebee5bd2ca6405611f1bae456db + languageName: node + linkType: hard + "refractor@npm:^4.8.0": version: 4.8.1 resolution: "refractor@npm:4.8.1" @@ -14341,6 +14527,13 @@ __metadata: languageName: node linkType: hard +"space-separated-tokens@npm:^1.0.0": + version: 1.1.5 + resolution: "space-separated-tokens@npm:1.1.5" + checksum: 10c0/3ee0a6905f89e1ffdfe474124b1ade9fe97276a377a0b01350bc079b6ec566eb5b219e26064cc5b7f3899c05bde51ffbc9154290b96eaf82916a1e2c2c13ead9 + languageName: node + linkType: hard + "space-separated-tokens@npm:^2.0.0": version: 2.0.2 resolution: "space-separated-tokens@npm:2.0.2" @@ -16232,6 +16425,13 @@ __metadata: languageName: node linkType: hard +"xtend@npm:^4.0.0": + version: 4.0.2 + resolution: "xtend@npm:4.0.2" + checksum: 10c0/366ae4783eec6100f8a02dff02ac907bf29f9a00b82ac0264b4d8b832ead18306797e283cf19de776538babfdcb2101375ec5646b59f08c52128ac4ab812ed0e + languageName: node + linkType: hard + "y18n@npm:^4.0.0": version: 4.0.3 resolution: "y18n@npm:4.0.3" diff --git a/openbas-model/src/main/java/io/openbas/database/model/ImportMapper.java b/openbas-model/src/main/java/io/openbas/database/model/ImportMapper.java new file mode 100644 index 0000000000..03b98f9cee --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/model/ImportMapper.java @@ -0,0 +1,84 @@ +package io.openbas.database.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.openbas.database.audit.ModelBaseListener; +import jakarta.persistence.*; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.hibernate.annotations.CreationTimestamp; +import org.hibernate.annotations.UpdateTimestamp; +import org.hibernate.annotations.UuidGenerator; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +@Data +@Entity +@Table(name = "import_mappers") +@EntityListeners(ModelBaseListener.class) +public class ImportMapper implements Base { + + @Id + @Column(name = "mapper_id") + @JsonProperty("import_mapper_id") + @GeneratedValue + @UuidGenerator + @NotNull + private UUID id; + + @Column(name = "mapper_name") + @JsonProperty("import_mapper_name") + @NotBlank + private String name; + + @Column(name = "mapper_inject_type_column") + @JsonProperty("import_mapper_inject_type_column") + @NotNull + private String injectTypeColumn; + + @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true, fetch = FetchType.EAGER) + @JoinColumn(name = "importer_mapper_id", nullable = false) + @JsonProperty("inject_importers") + private List injectImporters = new ArrayList<>(); + + @CreationTimestamp + @Column(name = "mapper_created_at") + @JsonProperty("import_mapper_created_at") + private Instant creationDate; + + @UpdateTimestamp + @Column(name = "mapper_updated_at") + @JsonProperty("import_mapper_updated_at") + private Instant updateDate; + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || !Base.class.isAssignableFrom(o.getClass())) { + return false; + } + Base base = (Base) o; + return id.equals(base.getId()); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String getId() { + return this.id != null ? this.id.toString() : ""; + } + + @Override + public void setId(String id) { + this.id = UUID.fromString(id); + } +} diff --git a/openbas-model/src/main/java/io/openbas/database/model/InjectImporter.java b/openbas-model/src/main/java/io/openbas/database/model/InjectImporter.java new file mode 100644 index 0000000000..610ffab944 --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/model/InjectImporter.java @@ -0,0 +1,84 @@ +package io.openbas.database.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import io.openbas.database.audit.ModelBaseListener; +import io.openbas.helper.MonoIdDeserializer; +import jakarta.persistence.*; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.hibernate.annotations.CreationTimestamp; +import org.hibernate.annotations.UpdateTimestamp; +import org.hibernate.annotations.UuidGenerator; + +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +@Data +@Entity +@Table(name = "inject_importers") +@EntityListeners(ModelBaseListener.class) +public class InjectImporter implements Base { + + @Id + @Column(name = "importer_id") + @JsonProperty("inject_importer_id") + @GeneratedValue + @UuidGenerator + @NotNull + private UUID id; + + @Column(name = "importer_import_type_value") + @JsonProperty("inject_importer_type_value") + @NotBlank + private String importTypeValue; + + @OneToOne + @JoinColumn(name = "importer_injector_contract_id") + @JsonProperty("inject_importer_injector_contract") + @JsonSerialize(using = MonoIdDeserializer.class) + @NotNull + private InjectorContract injectorContract; + + @OneToMany(cascade = CascadeType.ALL, orphanRemoval = true, fetch = FetchType.EAGER) + @JoinColumn(name = "attribute_inject_importer_id", nullable = false) + @JsonProperty("rule_attributes") + private List ruleAttributes = new ArrayList<>(); + + @CreationTimestamp + @Column(name="importer_created_at") + @JsonProperty("inject_importer_created_at") + private Instant creationDate; + + @UpdateTimestamp + @Column(name= "importer_updated_at") + @JsonProperty("inject_importer_updated_at") + private Instant updateDate; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || !Base.class.isAssignableFrom(o.getClass())) return false; + Base base = (Base) o; + return id.equals(base.getId()); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String getId() { + return this.id != null ? this.id.toString(): ""; + } + + @Override + public void setId(String id) { + this.id = UUID.fromString(id); + } +} diff --git a/openbas-model/src/main/java/io/openbas/database/model/InjectorContract.java b/openbas-model/src/main/java/io/openbas/database/model/InjectorContract.java index 48244ea047..e0c45cac79 100644 --- a/openbas-model/src/main/java/io/openbas/database/model/InjectorContract.java +++ b/openbas-model/src/main/java/io/openbas/database/model/InjectorContract.java @@ -13,6 +13,7 @@ import io.openbas.helper.MultiIdListDeserializer; import jakarta.persistence.*; import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import org.hibernate.annotations.Type; @@ -85,6 +86,7 @@ public class InjectorContract implements Base { @JsonSerialize(using = MonoIdDeserializer.class) @JsonProperty("injector_contract_injector") @Queryable(filterable = true, property = "id") + @NotNull private Injector injector; @Setter @@ -102,6 +104,11 @@ public class InjectorContract implements Base { @Queryable(filterable = true) private boolean isAtomicTesting; + @Column(name = "injector_contract_import_available") + @JsonProperty("injector_contract_import_available") + @Queryable(filterable = true) + private boolean isImportAvailable; + @JsonProperty("injector_contract_injector_type") private String getInjectorType() { return this.getInjector() != null ? this.getInjector().getType() : null; diff --git a/openbas-model/src/main/java/io/openbas/database/model/RuleAttribute.java b/openbas-model/src/main/java/io/openbas/database/model/RuleAttribute.java new file mode 100644 index 0000000000..2dcbb2a4e7 --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/model/RuleAttribute.java @@ -0,0 +1,84 @@ +package io.openbas.database.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.hypersistence.utils.hibernate.type.basic.PostgreSQLHStoreType; +import io.openbas.database.audit.ModelBaseListener; +import jakarta.persistence.*; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import org.hibernate.annotations.CreationTimestamp; +import org.hibernate.annotations.Type; +import org.hibernate.annotations.UpdateTimestamp; +import org.hibernate.annotations.UuidGenerator; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; + +@Data +@Entity +@Table(name = "rule_attributes") +@EntityListeners(ModelBaseListener.class) +public class RuleAttribute implements Base { + + @Id + @Column(name = "attribute_id") + @JsonProperty("rule_attribute_id") + @GeneratedValue + @UuidGenerator + @NotNull + private UUID id; + + @Column(name = "attribute_name") + @JsonProperty("rule_attribute_name") + @NotBlank + private String name; + + @Column(name = "attribute_columns") + @JsonProperty("rule_attribute_columns") + private String columns; + + @Column(name = "attribute_default_value") + @JsonProperty("rule_attribute_default_value") + private String defaultValue; + + @Type(PostgreSQLHStoreType.class) + @Column(name = "attribute_additional_config") + @JsonProperty("rule_attribute_additional_config") + private Map additionalConfig; + + @CreationTimestamp + @Column(name="attribute_created_at") + @JsonProperty("rule_attribute_created_at") + private Instant creationDate; + + @UpdateTimestamp + @Column(name= "attribute_updated_at") + @JsonProperty("rule_attribute_updated_at") + private Instant updateDate; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || !Base.class.isAssignableFrom(o.getClass())) return false; + Base base = (Base) o; + return id.equals(base.getId()); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String getId() { + return this.id != null ? this.id.toString(): ""; + } + + @Override + public void setId(String id) { + this.id = UUID.fromString(id); + } +} diff --git a/openbas-model/src/main/java/io/openbas/database/raw/RawPaginationImportMapper.java b/openbas-model/src/main/java/io/openbas/database/raw/RawPaginationImportMapper.java new file mode 100644 index 0000000000..ca8da14c36 --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/raw/RawPaginationImportMapper.java @@ -0,0 +1,24 @@ +package io.openbas.database.raw; + +import io.openbas.database.model.ImportMapper; +import jakarta.validation.constraints.NotBlank; +import lombok.Data; + +import java.time.Instant; + +@Data +public class RawPaginationImportMapper { + + @NotBlank + String import_mapper_id; + String import_mapper_name; + Instant import_mapper_created_at; + Instant import_mapper_updated_at; + + public RawPaginationImportMapper(final ImportMapper importMapper) { + this.import_mapper_id = importMapper.getId(); + this.import_mapper_name = importMapper.getName(); + this.import_mapper_created_at = importMapper.getCreationDate(); + this.import_mapper_updated_at = importMapper.getUpdateDate(); + } +} diff --git a/openbas-model/src/main/java/io/openbas/database/repository/ImportMapperRepository.java b/openbas-model/src/main/java/io/openbas/database/repository/ImportMapperRepository.java new file mode 100644 index 0000000000..7c4d95a50a --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/repository/ImportMapperRepository.java @@ -0,0 +1,18 @@ +package io.openbas.database.repository; + +import io.openbas.database.model.ImportMapper; +import org.jetbrains.annotations.NotNull; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.Pageable; +import org.springframework.data.jpa.domain.Specification; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.UUID; + +@Repository +public interface ImportMapperRepository extends CrudRepository { + + @NotNull + Page findAll(@NotNull Specification spec, @NotNull Pageable pageable); +} diff --git a/openbas-model/src/main/java/io/openbas/database/repository/InjectImporterRepository.java b/openbas-model/src/main/java/io/openbas/database/repository/InjectImporterRepository.java new file mode 100644 index 0000000000..6db7fc68eb --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/repository/InjectImporterRepository.java @@ -0,0 +1,11 @@ +package io.openbas.database.repository; + +import io.openbas.database.model.InjectImporter; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.UUID; + +@Repository +public interface InjectImporterRepository extends CrudRepository { +} diff --git a/openbas-model/src/main/java/io/openbas/database/repository/RuleAttributeRepository.java b/openbas-model/src/main/java/io/openbas/database/repository/RuleAttributeRepository.java new file mode 100644 index 0000000000..3f39e7d96a --- /dev/null +++ b/openbas-model/src/main/java/io/openbas/database/repository/RuleAttributeRepository.java @@ -0,0 +1,11 @@ +package io.openbas.database.repository; + +import io.openbas.database.model.RuleAttribute; +import org.springframework.data.repository.CrudRepository; +import org.springframework.stereotype.Repository; + +import java.util.UUID; + +@Repository +public interface RuleAttributeRepository extends CrudRepository { +}