From 046c85c1347cef4af4d64a030c80d032884695bf Mon Sep 17 00:00:00 2001 From: Marian Novotny <46998027+mariannovotny@users.noreply.github.com> Date: Wed, 8 Sep 2021 15:08:04 +0100 Subject: [PATCH] config editor rest: minor improvements in sigma rule importer (#313) * config editor rest: minor improvements in sigma rule importer * minor fixes * simplification of normalising name function --- alerting/alerting-core/pom.xml | 4 ++-- alerting/alerting-spark/pom.xml | 4 ++-- alerting/alerting-storm/pom.xml | 4 ++-- alerting/pom.xml | 2 +- config-editor/config-editor-core/pom.xml | 4 ++-- .../common/ConfigEditorUtils.java | 4 ++-- config-editor/config-editor-rest/pom.xml | 18 ++++++++--------- config-editor/config-editor-services/pom.xml | 20 +++++++++---------- .../alerts/sigma/SigmaRuleImporter.java | 4 +++- .../alerts/sigma/SigmaRuleImporterTest.java | 16 ++++++++++++--- config-editor/config-editor-sync/pom.xml | 8 ++++---- config-editor/pom.xml | 2 +- deployment/storm-topology-manager/pom.xml | 4 ++-- enriching/enriching-core/pom.xml | 6 +++--- enriching/enriching-storm/pom.xml | 4 ++-- enriching/pom.xml | 2 +- parsing/parsing-app/pom.xml | 6 +++--- parsing/parsing-core/pom.xml | 4 ++-- parsing/parsing-storm/pom.xml | 4 ++-- parsing/pom.xml | 2 +- pom.xml | 2 +- responding/pom.xml | 2 +- responding/responding-core/pom.xml | 6 +++--- responding/responding-stream/pom.xml | 6 +++--- siembol-common/pom.xml | 2 +- .../common/utils/EvaluationLibrary.java | 15 +++++++++----- 26 files changed, 86 insertions(+), 69 deletions(-) diff --git a/alerting/alerting-core/pom.xml b/alerting/alerting-core/pom.xml index a7bd48045..d181a4759 100644 --- a/alerting/alerting-core/pom.xml +++ b/alerting/alerting-core/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol alerting - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -35,7 +35,7 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.adrianwalker diff --git a/alerting/alerting-spark/pom.xml b/alerting/alerting-spark/pom.xml index 166855ea8..491663c36 100644 --- a/alerting/alerting-spark/pom.xml +++ b/alerting/alerting-spark/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol alerting - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -23,7 +23,7 @@ uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT jackson-databind diff --git a/alerting/alerting-storm/pom.xml b/alerting/alerting-storm/pom.xml index b052226bc..aa1aed495 100644 --- a/alerting/alerting-storm/pom.xml +++ b/alerting/alerting-storm/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol alerting - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -46,7 +46,7 @@ uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j diff --git a/alerting/pom.xml b/alerting/pom.xml index 6216d802c..e8e1480d6 100644 --- a/alerting/pom.xml +++ b/alerting/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT alerting-core diff --git a/config-editor/config-editor-core/pom.xml b/config-editor/config-editor-core/pom.xml index 0ea3b7ab8..8b31e2596 100644 --- a/config-editor/config-editor-core/pom.xml +++ b/config-editor/config-editor-core/pom.xml @@ -9,13 +9,13 @@ uk.co.gresearch.siembol config-editor - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.apache.commons diff --git a/config-editor/config-editor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java b/config-editor/config-editor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java index 35adc16aa..10f6719f1 100644 --- a/config-editor/config-editor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java +++ b/config-editor/config-editor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java @@ -27,6 +27,7 @@ public class ConfigEditorUtils { private static final String CONFIG_NAME_FORMAT = "%s.json"; private static final String TEST_CASE_NAME_FORMAT = "%s-%s.json"; private static final String TEST_CASE_NAME_PREFIX = "%s-"; + private static final String INVALID_NAME_SEQUENCE_REGEX ="[^a-zA-Z0-9_\\-]+"; static { Configuration.setDefaults(new Configuration.Defaults() { @@ -126,7 +127,6 @@ public static String getTestCaseFileNamePrefix(String configName) { } public static String getNormalisedConfigName(String configName) { - return configName.trim().replaceAll(" ", "_"); + return configName.trim().replaceAll(INVALID_NAME_SEQUENCE_REGEX, "_"); } - } diff --git a/config-editor/config-editor-rest/pom.xml b/config-editor/config-editor-rest/pom.xml index 0549de923..4af6adeb3 100644 --- a/config-editor/config-editor-rest/pom.xml +++ b/config-editor/config-editor-rest/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol config-editor - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -56,7 +56,7 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j @@ -67,22 +67,22 @@ uk.co.gresearch.siembol config-editor-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol config-editor-services - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol config-editor-sync - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j @@ -93,7 +93,7 @@ uk.co.gresearch.siembol parsing-app - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j @@ -104,7 +104,7 @@ uk.co.gresearch.siembol enriching-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j @@ -115,7 +115,7 @@ uk.co.gresearch.siembol responding-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j diff --git a/config-editor/config-editor-services/pom.xml b/config-editor/config-editor-services/pom.xml index 080fdf00a..f15f29e0b 100644 --- a/config-editor/config-editor-services/pom.xml +++ b/config-editor/config-editor-services/pom.xml @@ -10,53 +10,53 @@ uk.co.gresearch.siembol config-editor - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol config-editor-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol alerting-storm - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol parsing-storm - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol enriching-storm - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol parsing-app - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol enriching-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol responding-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT junit diff --git a/config-editor/config-editor-services/src/main/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java b/config-editor/config-editor-services/src/main/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java index 9e420083f..0dc4540aa 100644 --- a/config-editor/config-editor-services/src/main/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java +++ b/config-editor/config-editor-services/src/main/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java @@ -50,6 +50,7 @@ public class SigmaRuleImporter implements ConfigImporter { private static final String ERROR_IMPORT_CONFIG_LOG = "Error during importing sigma rule: {}, " + "attributes: {}, user:{}, exception: {}"; private static final String ERROR_TOKENS_PARSING = "Problem during parsing of condition tokens"; + private static final String RULE_UNKNOWN_FIELD_VALUE = "unknown"; private final String importerAttributesSchema; private final SiembolJsonSchemaValidator importerAttributesValidator; @@ -105,7 +106,8 @@ public ConfigEditorResult importConfig(UserInfo user, String importerAttributes, private RuleDto createRule(SigmaImporterAttributesDto attributes, Map sigmaRuleMap) throws Exception { RuleDto ret = new RuleDto(); BeanUtils.copyProperties(ret, attributes.getRuleMetadataMapping()); - EvaluationLibrary.substituteBean(ret, sigmaRuleMap); + EvaluationLibrary.substituteBean(ret, sigmaRuleMap, RULE_UNKNOWN_FIELD_VALUE); + ret.setRuleName(ConfigEditorUtils.getNormalisedConfigName(ret.getRuleName())); return ret; } diff --git a/config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java b/config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java index 5ae7d6d1c..b556440f1 100644 --- a/config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java +++ b/config-editor/config-editor-services/src/test/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java @@ -43,7 +43,7 @@ public class SigmaRuleImporterTest { private static String importerAttributes; /** - * title: Sigma Title + * title: Sigma Title( Experimental??? * id: d06be400-8045-4200-0067-740a2009db25 * status: experimental * description: Detects secret @@ -130,7 +130,7 @@ public void importConfig() throws JsonProcessingException { RuleDto rule = ALERTING_RULE_READER.readValue(result.getAttributes().getImportedConfiguration()); Assert.assertEquals("siembol", rule.getRuleAuthor()); - Assert.assertEquals("based_on_Sigma_Title", rule.getRuleName()); + Assert.assertEquals("based_on_Sigma_Title_Experimental_", rule.getRuleName()); Assert.assertEquals("generated from Detects secret and id: d06be400-8045-4200-0067-740a2009db25", rule.getRuleDescription()); Assert.assertEquals(0, rule.getRuleVersion()); @@ -187,13 +187,23 @@ public void importConfig() throws JsonProcessingException { } @Test - public void importConfigWithCondition1() throws JsonProcessingException { + public void importConfigWithCondition1() { String rule = sigmaRuleExample.replace("image_path and cmd_c and (cmd_s or not net_utility)", "(image_path or cmd_c) and (cmd_s or not net_utility)"); ConfigEditorResult result = importer.importConfig(userInfo, importerAttributes, rule); Assert.assertEquals(OK, result.getStatusCode()); } + @Test + public void importConfigWithUnknownFields() { + String rule = sigmaRuleExample.replace("description:", + "abc:"); + ConfigEditorResult result = importer.importConfig(userInfo, importerAttributes, rule); + Assert.assertTrue(result.getAttributes() + .getImportedConfiguration().contains("\"rule_description\" : \"unknown\"")); + Assert.assertEquals(OK, result.getStatusCode()); + } + @Test public void importConfigInvalidAttributes() { ConfigEditorResult result = importer.importConfig(userInfo, diff --git a/config-editor/config-editor-sync/pom.xml b/config-editor/config-editor-sync/pom.xml index f700907e7..faa4106f1 100644 --- a/config-editor/config-editor-sync/pom.xml +++ b/config-editor/config-editor-sync/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol config-editor - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -20,17 +20,17 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol config-editor-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol parsing-app - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT provided diff --git a/config-editor/pom.xml b/config-editor/pom.xml index 448ad6e62..c10df45bd 100644 --- a/config-editor/pom.xml +++ b/config-editor/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT config-editor-core diff --git a/deployment/storm-topology-manager/pom.xml b/deployment/storm-topology-manager/pom.xml index af18616ec..def7277d8 100644 --- a/deployment/storm-topology-manager/pom.xml +++ b/deployment/storm-topology-manager/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT ../../pom.xml @@ -43,7 +43,7 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j diff --git a/enriching/enriching-core/pom.xml b/enriching/enriching-core/pom.xml index 4eb6127e2..8529ff930 100644 --- a/enriching/enriching-core/pom.xml +++ b/enriching/enriching-core/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol enriching - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -35,12 +35,12 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.adrianwalker diff --git a/enriching/enriching-storm/pom.xml b/enriching/enriching-storm/pom.xml index af3a894a4..5fa7beb6b 100644 --- a/enriching/enriching-storm/pom.xml +++ b/enriching/enriching-storm/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol enriching - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -62,7 +62,7 @@ uk.co.gresearch.siembol enriching-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j diff --git a/enriching/pom.xml b/enriching/pom.xml index 741d75776..48de7fb08 100644 --- a/enriching/pom.xml +++ b/enriching/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT enriching-core diff --git a/parsing/parsing-app/pom.xml b/parsing/parsing-app/pom.xml index 06c164cbf..988f92ca1 100644 --- a/parsing/parsing-app/pom.xml +++ b/parsing/parsing-app/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol parsing - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -39,12 +39,12 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol parsing-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.adrianwalker diff --git a/parsing/parsing-core/pom.xml b/parsing/parsing-core/pom.xml index dc0583fb6..f95d64307 100644 --- a/parsing/parsing-core/pom.xml +++ b/parsing/parsing-core/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol parsing - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -45,7 +45,7 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT joda-time diff --git a/parsing/parsing-storm/pom.xml b/parsing/parsing-storm/pom.xml index fda76c07f..388befee4 100644 --- a/parsing/parsing-storm/pom.xml +++ b/parsing/parsing-storm/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol parsing - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -62,7 +62,7 @@ uk.co.gresearch.siembol parsing-app - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j diff --git a/parsing/pom.xml b/parsing/pom.xml index 525d24dfe..dcfd4c039 100644 --- a/parsing/pom.xml +++ b/parsing/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT parsing-core diff --git a/pom.xml b/pom.xml index fd6df9484..0a636d4b9 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ uk.co.gresearch.siembol siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT A scalable, advanced security analytics framework based on open-source big data technologies. 2019 https://siembol.io/ diff --git a/responding/pom.xml b/responding/pom.xml index cf60e0b27..3bba7c6fa 100644 --- a/responding/pom.xml +++ b/responding/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT responding-core diff --git a/responding/responding-core/pom.xml b/responding/responding-core/pom.xml index e1e6f8973..d72821ade 100644 --- a/responding/responding-core/pom.xml +++ b/responding/responding-core/pom.xml @@ -11,7 +11,7 @@ uk.co.gresearch.siembol responding - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -35,12 +35,12 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT uk.co.gresearch.siembol alerting-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT com.jayway.jsonpath diff --git a/responding/responding-stream/pom.xml b/responding/responding-stream/pom.xml index c4031b2b3..ede8db67c 100644 --- a/responding/responding-stream/pom.xml +++ b/responding/responding-stream/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol responding - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT @@ -51,7 +51,7 @@ uk.co.gresearch.siembol siembol-common - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.slf4j @@ -62,7 +62,7 @@ uk.co.gresearch.siembol responding-core - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT org.apache.kafka diff --git a/siembol-common/pom.xml b/siembol-common/pom.xml index 8debf4dae..43f80e325 100644 --- a/siembol-common/pom.xml +++ b/siembol-common/pom.xml @@ -9,7 +9,7 @@ uk.co.gresearch.siembol siembol - 1.3.5-SNAPSHOT + 1.3.6-SNAPSHOT diff --git a/siembol-common/src/main/java/uk/co/gresearch/siembol/common/utils/EvaluationLibrary.java b/siembol-common/src/main/java/uk/co/gresearch/siembol/common/utils/EvaluationLibrary.java index e2fb2a6d4..439fe42b2 100644 --- a/siembol-common/src/main/java/uk/co/gresearch/siembol/common/utils/EvaluationLibrary.java +++ b/siembol-common/src/main/java/uk/co/gresearch/siembol/common/utils/EvaluationLibrary.java @@ -65,18 +65,23 @@ public static boolean containsVariables(String str) { return numVariableMatches == numVariableStartMatches && numVariableMatches > 0; } - @SuppressWarnings({"rawtypes", "unchecked"}) + public static Object substituteBean(Object obj, Map event) throws Exception { + return substituteBean(obj, event, null); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + public static Object substituteBean(Object obj, Map event, String defaultValue) throws Exception { //NOTE: currently we have beans with primitive types, Bean or List if (obj instanceof String) { - return substitute(event, (String)obj).orElse(null); + return substitute(event, (String)obj).orElse(defaultValue); } else if (obj instanceof Enum) { return obj; } else if (obj instanceof List) { List list = (List)obj; for (int i = 0; i < list.size(); i++) { - list.set(i, substituteBean(list.get(i), event)); + list.set(i, substituteBean(list.get(i), event, defaultValue)); } return list; } else if (obj != null) { @@ -92,12 +97,12 @@ else if (obj instanceof List) { PropertyUtils.setNestedProperty( obj, fieldName, - substituteBean(property, event)); + substituteBean(property, event, defaultValue)); } } return obj; } - return null; + return defaultValue; } public static Optional cloneAndSubstituteBean(Object prototype,