config editor rest: minor improvements in sigma rule importer (#313)

* config editor rest: minor improvements in sigma rule importer * minor fixes * simplification of normalising name function
G-Research · Sep 8, 2021 · 046c85c · 046c85c
1 parent 67d7445
commit 046c85c
Show file tree

Hide file tree

Showing 26 changed files with 86 additions and 69 deletions.
diff --git a/alerting/alerting-core/pom.xml b/alerting/alerting-core/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>alerting</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -35,7 +35,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.adrianwalker</groupId>

diff --git a/alerting/alerting-spark/pom.xml b/alerting/alerting-spark/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>alerting</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -23,7 +23,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <artifactId>jackson-databind</artifactId>

diff --git a/alerting/alerting-storm/pom.xml b/alerting/alerting-storm/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>alerting</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -46,7 +46,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

diff --git a/alerting/pom.xml b/alerting/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>siembol</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <modules>
         <module>alerting-core</module>

diff --git a/config-editor/config-editor-core/pom.xml b/config-editor/config-editor-core/pom.xml
@@ -9,13 +9,13 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>config-editor</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>

diff --git a/...tor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java b/...tor-core/src/main/java/uk/co/gresearch/siembol/configeditor/common/ConfigEditorUtils.java
@@ -27,6 +27,7 @@ public class ConfigEditorUtils {
     private static final String CONFIG_NAME_FORMAT = "%s.json";
     private static final String TEST_CASE_NAME_FORMAT = "%s-%s.json";
     private static final String TEST_CASE_NAME_PREFIX = "%s-";
+    private static final String INVALID_NAME_SEQUENCE_REGEX ="[^a-zA-Z0-9_\\-]+";
 
     static {
         Configuration.setDefaults(new Configuration.Defaults() {
@@ -126,7 +127,6 @@ public static String getTestCaseFileNamePrefix(String configName) {
     }
 
     public static String getNormalisedConfigName(String configName) {
-        return configName.trim().replaceAll(" ", "_");
+        return configName.trim().replaceAll(INVALID_NAME_SEQUENCE_REGEX, "_");
     }
-
 }
diff --git a/config-editor/config-editor-rest/pom.xml b/config-editor/config-editor-rest/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>config-editor</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencyManagement>
         <dependencies>
@@ -56,7 +56,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -67,22 +67,22 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>config-editor-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>config-editor-services</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>config-editor-sync</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -93,7 +93,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>parsing-app</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -104,7 +104,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>enriching-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>
@@ -115,7 +115,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>responding-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

diff --git a/config-editor/config-editor-services/pom.xml b/config-editor/config-editor-services/pom.xml
@@ -10,53 +10,53 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>config-editor</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>config-editor-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-storm</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>parsing-storm</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>enriching-storm</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>parsing-app</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>enriching-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>responding-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>

diff --git a/...ain/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java b/...ain/java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporter.java
@@ -50,6 +50,7 @@ public class SigmaRuleImporter implements ConfigImporter {
     private static final String ERROR_IMPORT_CONFIG_LOG = "Error during importing sigma rule: {}, " +
             "attributes: {}, user:{}, exception: {}";
     private static final String ERROR_TOKENS_PARSING = "Problem during parsing of condition tokens";
+    private static final String RULE_UNKNOWN_FIELD_VALUE = "unknown";
 
     private final String importerAttributesSchema;
     private final SiembolJsonSchemaValidator importerAttributesValidator;
@@ -105,7 +106,8 @@ public ConfigEditorResult importConfig(UserInfo user, String importerAttributes,
     private RuleDto createRule(SigmaImporterAttributesDto attributes, Map<String, Object> sigmaRuleMap) throws Exception {
         RuleDto ret = new RuleDto();
         BeanUtils.copyProperties(ret, attributes.getRuleMetadataMapping());
-        EvaluationLibrary.substituteBean(ret, sigmaRuleMap);
+        EvaluationLibrary.substituteBean(ret, sigmaRuleMap, RULE_UNKNOWN_FIELD_VALUE);
+
         ret.setRuleName(ConfigEditorUtils.getNormalisedConfigName(ret.getRuleName()));
         return ret;
     }

diff --git a/...java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java b/...java/uk/co/gresearch/siembol/configeditor/service/alerts/sigma/SigmaRuleImporterTest.java
@@ -43,7 +43,7 @@ public class SigmaRuleImporterTest {
     private static String importerAttributes;
 
     /**
-     * title: Sigma Title
+     * title: Sigma Title( Experimental???
      * id: d06be400-8045-4200-0067-740a2009db25
      * status: experimental
      * description: Detects secret
@@ -130,7 +130,7 @@ public void importConfig() throws JsonProcessingException {
 
         RuleDto rule = ALERTING_RULE_READER.readValue(result.getAttributes().getImportedConfiguration());
         Assert.assertEquals("siembol", rule.getRuleAuthor());
-        Assert.assertEquals("based_on_Sigma_Title", rule.getRuleName());
+        Assert.assertEquals("based_on_Sigma_Title_Experimental_", rule.getRuleName());
         Assert.assertEquals("generated from Detects secret and id: d06be400-8045-4200-0067-740a2009db25",
                 rule.getRuleDescription());
         Assert.assertEquals(0, rule.getRuleVersion());
@@ -187,13 +187,23 @@ public void importConfig() throws JsonProcessingException {
     }
 
     @Test
-    public void importConfigWithCondition1() throws JsonProcessingException {
+    public void importConfigWithCondition1() {
         String rule  = sigmaRuleExample.replace("image_path and cmd_c and (cmd_s or not net_utility)",
                 "(image_path or cmd_c) and (cmd_s or not net_utility)");
         ConfigEditorResult result = importer.importConfig(userInfo, importerAttributes, rule);
         Assert.assertEquals(OK, result.getStatusCode());
     }
 
+    @Test
+    public void importConfigWithUnknownFields() {
+        String rule  = sigmaRuleExample.replace("description:",
+                "abc:");
+        ConfigEditorResult result = importer.importConfig(userInfo, importerAttributes, rule);
+        Assert.assertTrue(result.getAttributes()
+                .getImportedConfiguration().contains("\"rule_description\" : \"unknown\""));
+        Assert.assertEquals(OK, result.getStatusCode());
+    }
+
     @Test
     public void importConfigInvalidAttributes() {
         ConfigEditorResult result = importer.importConfig(userInfo,

diff --git a/config-editor/config-editor-sync/pom.xml b/config-editor/config-editor-sync/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>config-editor</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -20,17 +20,17 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>config-editor-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>parsing-app</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <scope>provided</scope>
         </dependency>
         <dependency>

diff --git a/config-editor/pom.xml b/config-editor/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>siembol</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <modules>
         <module>config-editor-core</module>

diff --git a/deployment/storm-topology-manager/pom.xml b/deployment/storm-topology-manager/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>siembol</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
         <relativePath>../../pom.xml</relativePath>
     </parent>
     <dependencyManagement>
@@ -43,7 +43,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>

diff --git a/enriching/enriching-core/pom.xml b/enriching/enriching-core/pom.xml
@@ -11,7 +11,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>enriching</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -35,12 +35,12 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>siembol-common</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>alerting-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
         </dependency>
         <dependency>
             <groupId>org.adrianwalker</groupId>

diff --git a/enriching/enriching-storm/pom.xml b/enriching/enriching-storm/pom.xml
@@ -9,7 +9,7 @@
     <parent>
         <groupId>uk.co.gresearch.siembol</groupId>
         <artifactId>enriching</artifactId>
-        <version>1.3.5-SNAPSHOT</version>
+        <version>1.3.6-SNAPSHOT</version>
     </parent>
     <dependencies>
         <dependency>
@@ -62,7 +62,7 @@
         <dependency>
             <groupId>uk.co.gresearch.siembol</groupId>
             <artifactId>enriching-core</artifactId>
-            <version>1.3.5-SNAPSHOT</version>
+            <version>1.3.6-SNAPSHOT</version>
             <exclusions>
                 <exclusion>
                     <groupId>org.slf4j</groupId>