From d823b1a94e03114e1a51fe2f00096ec2377f9df5 Mon Sep 17 00:00:00 2001 From: Gael Leblan Date: Wed, 28 Aug 2024 15:29:35 +0200 Subject: [PATCH] [frontend/backend] In XLS mapper, having a way to map "All teams" as targets of injects (#1349) --- .../InjectorContractApi.java | 1 + .../InjectorContractService.java | 30 ++++++++ .../io/openbas/service/InjectService.java | 36 +++++++--- .../src/main/resources/application.properties | 3 + .../src/test/resources/application.properties | 4 ++ .../xls_mapper/RulesContractContent.tsx | 72 +++++++++++++------ openbas-front/src/utils/Localization.js | 3 + 7 files changed, 119 insertions(+), 30 deletions(-) diff --git a/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractApi.java b/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractApi.java index 178abf246c..b18adcd0cd 100644 --- a/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractApi.java +++ b/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractApi.java @@ -39,6 +39,7 @@ public class InjectorContractApi extends RestBehavior { private final InjectorRepository injectorRepository; private final InjectorContractRepository injectorContractRepository; + private final InjectorContractService injectorContractService; @GetMapping("/api/injector_contracts") diff --git a/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractService.java b/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractService.java index aa3d545140..ad09bf4510 100644 --- a/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractService.java +++ b/openbas-api/src/main/java/io/openbas/rest/injector_contract/InjectorContractService.java @@ -1,6 +1,10 @@ package io.openbas.rest.injector_contract; import io.openbas.database.model.*; +import io.openbas.database.repository.InjectorContractRepository; +import io.openbas.injectors.email.EmailContract; +import io.openbas.injectors.email.EmailInjector; +import io.openbas.injectors.ovh.OvhSmsContract; import io.openbas.rest.injector_contract.output.InjectorContractOutput; import jakarta.persistence.EntityManager; import jakarta.persistence.PersistenceContext; @@ -10,12 +14,16 @@ import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; import org.springframework.data.domain.Page; import org.springframework.data.domain.PageImpl; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.domain.Specification; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; @@ -32,6 +40,28 @@ public class InjectorContractService { @PersistenceContext private EntityManager entityManager; + private final InjectorContractRepository injectorContractRepository; + + @Value("${openbas.xls.import.mail.enable}") + private boolean mailImportEnabled; + + @Value("${openbas.xls.import.sms.enable}") + private boolean smsImportEnabled; + + @EventListener(ApplicationReadyEvent.class) + public void initImportAvailableOnStartup() { + List listOfInjectorImportAvailable = new ArrayList<>(); + if (mailImportEnabled) listOfInjectorImportAvailable.addAll(Arrays.asList(EmailContract.EMAIL_GLOBAL, EmailContract.EMAIL_DEFAULT)); + if (smsImportEnabled) listOfInjectorImportAvailable.add(OvhSmsContract.OVH_DEFAULT); + + List listInjectorContract = new ArrayList<>(); + injectorContractRepository.findAll().spliterator().forEachRemaining(listInjectorContract::add); + listInjectorContract.forEach(injectorContract -> { + injectorContract.setImportAvailable(listOfInjectorImportAvailable.contains(injectorContract.getId())); + }); + injectorContractRepository.saveAll(listInjectorContract); + } + public Page injectorContracts( @Nullable final Specification specification, @NotNull final Pageable pageable) { diff --git a/openbas-api/src/main/java/io/openbas/service/InjectService.java b/openbas-api/src/main/java/io/openbas/service/InjectService.java index 8a068e67bf..0268f1c6b6 100644 --- a/openbas-api/src/main/java/io/openbas/service/InjectService.java +++ b/openbas-api/src/main/java/io/openbas/service/InjectService.java @@ -287,7 +287,7 @@ private ImportTestSummary importXls(String importId, Scenario scenario, ImportMa try { // We open the previously saved file String tmpdir = System.getProperty("java.io.tmpdir"); - java.nio.file.Path file = Files.list(Path.of(tmpdir, pathSeparator, importId, pathSeparator)).findFirst().orElseThrow(); + Path file = Files.list(Path.of(tmpdir, pathSeparator, importId, pathSeparator)).findFirst().orElseThrow(); // We open the file and convert it to an apache POI object InputStream xlsFile = Files.newInputStream(file); @@ -300,7 +300,17 @@ private ImportTestSummary importXls(String importId, Scenario scenario, ImportMa Map mapPatternByInjectImport = importMapper .getInjectImporters().stream().collect( Collectors.toMap(InjectImporter::getId, - injectImporter -> Pattern.compile(injectImporter.getImportTypeValue()) + injectImporter -> Pattern.compile(injectImporter.getImportTypeValue()) + )); + + Map mapPatternByAllTeams = importMapper.getInjectImporters().stream() + .flatMap(injectImporter -> injectImporter.getRuleAttributes().stream()) + .filter(ruleAttribute -> Objects.equals(ruleAttribute.getName(), "teams")) + .filter(ruleAttribute -> ruleAttribute.getAdditionalConfig() != null && !Strings.isBlank(ruleAttribute.getAdditionalConfig().get("allTeamsValue"))) + .collect( + Collectors.toMap(ruleAttribute -> ruleAttribute.getAdditionalConfig().get("allTeamsValue"), + ruleAttribute -> Pattern.compile(ruleAttribute.getAdditionalConfig().get("allTeamsValue")), + (first, second) -> first )); // We also get the list of teams into a map to be able to get them easily later on @@ -321,7 +331,7 @@ private ImportTestSummary importXls(String importId, Scenario scenario, ImportMa // For each rows of the selected sheet selectedSheet.rowIterator().forEachRemaining(row -> { - ImportRow rowSummary = importRow(row, importMapper, scenario, mapPatternByInjectImport, mapTeamByName, + ImportRow rowSummary = importRow(row, importMapper, scenario, mapPatternByInjectImport, mapTeamByName, mapPatternByAllTeams, zoneOffset); importTestSummary.getImportMessage().addAll(rowSummary.getImportMessages()); if(rowSummary.getInject() != null) { @@ -366,7 +376,7 @@ private ImportTestSummary importXls(String importId, Scenario scenario, ImportMa private ImportRow importRow(Row row, ImportMapper importMapper, Scenario scenario, Map mapPatternByInjectImport, Map mapTeamByName, - ZoneOffset timezoneOffset) { + Map mapPatternByAllTeams, ZoneOffset timezoneOffset) { ImportRow importTestSummary = new ImportRow(); // The column that differenciate the importer is the same for all so we get it right now int colTypeIdx = CellReference.convertColStringToIndex(importMapper.getInjectTypeColumn()); @@ -574,10 +584,8 @@ private ImportRow importRow(Row row, ImportMapper importMapper, Scenario scenari matchingInjectImporter.getRuleAttributes().forEach(ruleAttribute -> { importTestSummary.getImportMessages().addAll( addFields(inject, ruleAttribute, - row, mapTeamByName, expectation, importMapper)); + row, mapTeamByName, expectation, importMapper, mapPatternByAllTeams)); }); - // This is by default at false - inject.setAllTeams(false); // The user is the one doing the import inject.setUser(userRepository.findById(currentUser().getId()).orElseThrow()); // No exercise yet @@ -630,7 +638,8 @@ private void setAttributeValue(Row row, InjectImporter matchingInjectImporter, S private List addFields(Inject inject, RuleAttribute ruleAttribute, Row row, Map mapTeamByName, AtomicReference expectation, - ImportMapper importMapper) { + ImportMapper importMapper, + Map mapPatternByAllTeams) { // If it's a reserved field, it's already taken care of if(importReservedField.contains(ruleAttribute.getName())) { return Collections.emptyList(); @@ -668,8 +677,8 @@ private List addFields(Inject inject, RuleAttribute ruleAttribute case "team": // If the rule type is on a team field, we split by "+" if there is a concatenation of columns // and then joins the result, split again by "," and use the list of results to get the teams by their name - List columnValues = new ArrayList<>(); + String allTeamsValue = ruleAttribute.getAdditionalConfig() != null ? ruleAttribute.getAdditionalConfig().get("allTeamsValue") : null; if(ruleAttribute.getColumns() != null) { columnValues = Arrays.stream(Arrays.stream(ruleAttribute.getColumns().split("\\+")) .map(column -> getValueAsString(row, column)) @@ -687,7 +696,14 @@ private List addFields(Inject inject, RuleAttribute ruleAttribute } else { List importMessages = new ArrayList<>(); columnValues.forEach(teamName -> { - if(mapTeamByName.containsKey(teamName)) { + inject.setAllTeams(false); + Matcher allTeamsMatcher = null; + if(mapPatternByAllTeams.get(allTeamsValue) != null) { + allTeamsMatcher = mapPatternByAllTeams.get(allTeamsValue).matcher(teamName); + } + if (allTeamsValue != null && allTeamsMatcher != null && allTeamsMatcher.find()) { + inject.setAllTeams(true); + } else if(mapTeamByName.containsKey(teamName)) { inject.getTeams().add(mapTeamByName.get(teamName)); } else { // The team does not exist, we create a new one diff --git a/openbas-api/src/main/resources/application.properties b/openbas-api/src/main/resources/application.properties index ab85077a82..92d6463182 100644 --- a/openbas-api/src/main/resources/application.properties +++ b/openbas-api/src/main/resources/application.properties @@ -194,6 +194,9 @@ openbas.mail.imap.starttls.enable=false openbas.xtm.opencti.enable=false openbas.xtm.opencti.url= openbas.xtm.opencti.token= +# XLS Import +openbas.xls.import.mail.enable=true +openbas.xls.import.sms.enable=true # Injector Caldera config injector.caldera.enable=true diff --git a/openbas-api/src/test/resources/application.properties b/openbas-api/src/test/resources/application.properties index 66a96c4358..4d53838651 100644 --- a/openbas-api/src/test/resources/application.properties +++ b/openbas-api/src/test/resources/application.properties @@ -76,6 +76,10 @@ http.enable=false # Injector Caldera config injector.caldera.enable=false +# XLS Import +openbas.xls.import.mail.enable=true +openbas.xls.import.sms.enable=true + ############# # COLLECTORS # ############# diff --git a/openbas-front/src/admin/components/settings/data_ingestion/xls_mapper/RulesContractContent.tsx b/openbas-front/src/admin/components/settings/data_ingestion/xls_mapper/RulesContractContent.tsx index b13e422898..61717dbaf8 100644 --- a/openbas-front/src/admin/components/settings/data_ingestion/xls_mapper/RulesContractContent.tsx +++ b/openbas-front/src/admin/components/settings/data_ingestion/xls_mapper/RulesContractContent.tsx @@ -233,6 +233,36 @@ const RulesContractContent: React.FC = ({ )} /> {rulesFields.map((ruleField, rulesIndex) => { + let cogIcon; + if (ruleField.rule_attribute_name === 'trigger_time') { + cogIcon = + + ; + } else if (ruleField.rule_attribute_name === 'teams') { + cogIcon = + + ; + } else { + cogIcon = + + ; + } return (
@@ -261,26 +291,7 @@ const RulesContractContent: React.FC = ({ color="primary" onClick={() => handleDefaultValueOpen(rulesIndex)} > - {(ruleField.rule_attribute_name === 'trigger_time') - ? ( - - - - ) : ( - - - ) - } + {cogIcon}
{currentRuleIndex !== null @@ -320,6 +331,27 @@ const RulesContractContent: React.FC = ({
} + {currentRuleIndex === rulesFields.findIndex((r) => r.rule_attribute_name === 'teams') + &&
+ + + + +
+ }