fixes to create working ecs config

ecs-configuration/elasticsearch/ecs-fields-tls.json

@@ -37,8 +37,16 @@
                 "type": "keyword"
               },
               "ja3": {
-                "ignore_above": 1024,
-                "type": "keyword"
+                "properties": {
+                  "hash": { 
+                    "ignore_above": 8000,
+                    "type": "keyword"
+                  },
+                  "string" : {
+                    "ignore_above": 1024,
+                    "type": "keyword"
+                  }
+                }
               },
               "server_name": {
                 "ignore_above": 8000,
@@ -98,8 +106,16 @@
                 "type": "keyword"
               },
               "ja3s": {
-                "ignore_above": 1024,
-                "type": "keyword"
+                "properties": {
+                  "hash": {
+                    "ignore_above": 8000,
+                    "type": "keyword"
+                  },
+                  "string": {
+                    "ignore_above": 8000,
+                    "type": "keyword"
+                  }
+                }
               },
               "subject": {
                 "ignore_above": 8000,

ecs-configuration/elasticsearch/ecs-zeek-network-ssh.json

@@ -36,13 +36,14 @@
             "type": "keyword"
           },
           "client": {
-            "ignore_above": 8000,
-            "fields": {
-              "text": {
+            "properties": {
+              "version": {
+                "type": "text"
+              },
+              "software": {
                 "type": "text"
               }
-            },
-            "type": "keyword"
+            }
           },
           "compression_algorithm": {
             "ignore_above": 2048,

ecs-configuration/logstash/conf.d/logstash-100-input-kafka-fsf.conf

@@ -5,8 +5,7 @@ input {
     # Set this to one per kafka partition to scale up
     #consumer_threads => 4
     group_id => "fsf_logstash"
-    bootstrap_servers => "127.0.0.1:9092"
-    codec => json
+    bootstrap_servers => "simplerockbuild.simplerock.lan:9092"    codec => json
     auto_offset_reset => "earliest"
     id => "input-kafka-fsf"
   }

ecs-configuration/logstash/conf.d/logstash-502-filter-zeek-network-ssh.conf

@@ -4,7 +4,10 @@ filter {
     if [ssh] {
 
       mutate {
-        rename => { "[ssh][direction]" => "[network][direction]" }
+        rename => { 
+          "[ssh][direction]" => "[network][direction]" 
+          "[ssh][client]" => "[ssh][client][software]"
+        }
       }
 
       # Event outcome
@@ -23,4 +26,4 @@ filter {
     }
 
   }
-}
+}

ecs-configuration/logstash/conf.d/logstash-502-filter-zeek-network-ssl.conf

@@ -13,8 +13,8 @@ filter {
           "[tls][client_issuer]" => "[tls][client][issuer]"
           "[tls][client_subject]" => "[tls][client][subject]"
           "[tls][issuer]" => "[tls][server][issuer]"
-          "[tls][ja3]" => "[tls][client][ja3]"
-          "[tls][ja3s]" => "[tls][server][ja3s]"
+          "[tls][ja3]" => "[tls][client][ja3][hash]"
+          "[tls][ja3s]" => "[tls][server][ja3s][hash]"
           "[tls][orig_certificate_sha1]" => "[tls][client][hash][sha1]"
           "[tls][resp_certificate_sha1]" => "[tls][server][hash][sha1]"
           "[tls][server_name]" => "[tls][client][server_name]"
@@ -46,4 +46,4 @@ filter {
 
     }
   }
-}
+}

ecs-configuration/logstash/conf.d/logstash-599-filter-zeek-hook.conf

@@ -9,15 +9,15 @@ filter {
     }
 
     # Duration is always float (milliseconds) in any of the log types, therefore apply millisecond to nanosecond for event duration if it exists
-    if [event][duration] {
-      ruby {
-        code => "
-          event_duration = event.get('[event][duration]')
-          event.set('[event][duration]', event_duration * 1000000)
-        "
-        tag_on_exception => "_rubyexception-zeek-all-duration_to_nanoseconds"
-      }
-    }
+#    if [event][duration] {
+#      ruby {
+#        code => "
+#          event_duration = event.get('[event][duration]')
+#          event.set('[event][duration]', event_duration * 1000000)
+#        "
+#        tag_on_exception => "_rubyexception-zeek-all-duration_to_nanoseconds"
+#      }
+#    }
 
   }
 }

ecs-configuration/logstash/conf.d/logstash-601-filter-suricata-ssh.conf

@@ -4,12 +4,16 @@ filter {
     if [ssh] {
       # rename nested ssh otherwise name collisions
       mutate {
-        rename => { "[ssh][server]" => "[suricata][ssh]" }
+        rename => { 
+          "[ssh][server]" => "[suricata][ssh]"       
+        }
       }
       mutate {
         rename => {
-          "[suricata][ssh][server][proto_version]" => "[ssh][version]"
-          "[suricata][ssh][server][software_version]" => "[ssh][server]"
+          "[suricata][ssh][server][proto_version]" => "[ssh][server][version]"
+          "[suricata][ssh][server][software_version]" => "[ssh][server][software]"
+          "[suricata][ssh][client][proto_version]" => "[ssh][client][version]"
+          "[suricata][ssh][client][software_version]" => "[ssh][client][software]"
         }
       }
     }

ecs-configuration/logstash/conf.d/logstash-601-filter-suricata-tls.conf

@@ -7,8 +7,10 @@ filter {
           "[tls][chain]" => "[tls][server][certificate_chain]"
           "[tls][fingerprint]" => "[tls][server][hash][sha1]"
           "[tls][issuer]" => "[tls][server][issuer]"
-          "[tls][ja3]" => "[tls][client][ja3]"
-          "[tls][ja3s]" => "[tls][server][ja3s]"
+          "[tls][ja3][hash]" => "[tls][client][ja3][hash]"
+          "[tls][ja3][string]" => "[tls][client][ja3][string]"
+          "[tls][ja3s][hash]" => "[tls][server][ja3s][hash]"
+          "[tls][ja3s][string]" => "[tls][server][ja3s][string]"
           "[tls][not_after]" => "[tls][server][not_after]"
           "[tls][not_before]" => "[tls][server][not_before]"
           "[tls][session_resumed]" => "[tls][resumed]"
@@ -19,4 +21,4 @@ filter {
       }
 
   }
-}
+}

ecs-configuration/logstash/conf.d/logstash-9999-output-elasticsearch.conf

@@ -1,13 +1,12 @@
 output {
-
   # Requires event module and category
   if [event][module] and [event][category] {
 
     # Requires event dataset
     if [event][dataset] {
       elasticsearch {
-          hosts => [ "127.0.0.1" ]
-          index => "ecs-%{[event][module]}-%{[event][category]}-%{+YYYY.MM.dd}"
+                    hosts => ["127.0.0.1:9200"]
+                    index => "ecs-%{[event][module]}-%{[event][category]}-%{+YYYY.MM.dd}"
           manage_template => false
       }
     }
@@ -16,15 +15,15 @@ output {
       # Suricata or Zeek JSON error possibly, ie: Suricata without a event.dataset seen with filebeat error, but doesn't have a tag
       if [event][module] == "suricata" or [event][module] == "zeek" {
         elasticsearch {
-            hosts => [ "127.0.0.1" ]
-            index => "parse-failures-%{+YYYY.MM.dd}"
+                        hosts => ["127.0.0.1:9200"]
+                        index => "parse-failures-%{+YYYY.MM.dd}"
             manage_template => false
         }
       }
       else {
         elasticsearch {
-            hosts => [ "127.0.0.1" ]
-            index => "ecs-%{[event][module]}-%{[event][category]}-%{+YYYY.MM.dd}"
+                        hosts => ["127.0.0.1:9200"]
+                        index => "ecs-%{[event][module]}-%{[event][category]}-%{+YYYY.MM.dd}"
             manage_template => false
         }
       }
@@ -33,28 +32,27 @@ output {
 
   else if [@metadata][stage] == "fsfraw_kafka" {
     elasticsearch {
-        hosts => [ "127.0.0.1" ]
-        index => "fsf-%{+YYYY.MM.dd}"
+                hosts => ["127.0.0.1:9200"]
+                index => "fsf-%{+YYYY.MM.dd}"
         manage_template => false
     }
-
   }
 
   else if [@metadata][stage] == "_parsefailure" {
     elasticsearch {
-        hosts => [ "127.0.0.1" ]
-        index => "parse-failures-%{+YYYY.MM.dd}"
+                hosts => ["127.0.0.1:9200"]
+                index => "parse-failures-%{+YYYY.MM.dd}"
         manage_template => false
     }
+
   }
 
   # Catch all index that is not RockNSM or ECS or parse failures
   else {
     elasticsearch {
-        hosts => [ "127.0.0.1" ]
-        index => "indexme-%{+YYYY.MM.dd}"
+                hosts => ["127.0.0.1:9200"]
+                index => "indexme-%{+YYYY.MM.dd}"
         manage_template => false
     }
   }
-
 }