Merge pull request #97 from satta/nullfix

fix parsing of JSON null values

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 All notable changes to FEVER will be documented in this file.
 
+## [1.3.3] - 2022-01-25
+
+### Changed
+- Fixed handling of JSON `null` values (#97)
+
 ## [1.3.2] - 2021-12-09
 
 ### Added

util/testdata/jsonparse_eve_nulls.json

@@ -0,0 +1 @@
+{"timestamp":"2017-03-06T06:54:10.839668+0000","flow_id":null,"in_iface":"enp2s0f1","event_type":"fileinfo","vlan":null,"src_ip":null,"src_port":null,"dest_ip":null,"dest_port":null,"http":{"hostname":"api.icndb.com","url":null,"state":"CLOSED","md5":null}}

util/util.go

@@ -4,6 +4,7 @@ package util
 // Copyright (c) 2017, 2018, 2020, DCSO GmbH
 
 import (
+	"bytes"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
@@ -73,6 +74,11 @@ func ParseJSON(json []byte) (e types.Entry, parseerr error) {
 			parseerr = err
 			return
 		}
+		// skip null fields; these will not be handled by the low-level
+		// jsonparser.Parse* () functions
+		if bytes.Equal(value, []byte("null")) {
+			return
+		}
 		switch idx {
 		case 0:
 			e.EventType, err = jsonparser.ParseString(value)

util/util_test.go

@@ -12,6 +12,14 @@ import (
 	"github.com/DCSO/fever/types"
 )
 
+var nullEntry = types.Entry{
+	Timestamp: "2017-03-06T06:54:10.839668+0000",
+	EventType: "fileinfo",
+	JSONLine:  `{"timestamp":"2017-03-06T06:54:10.839668+0000","flow_id":null,"in_iface":"enp2s0f1","event_type":"fileinfo","vlan":null,"src_ip":null,"src_port":null,"dest_ip":null,"dest_port":null,"http":{"hostname":"api.icndb.com","url":null,"state":"CLOSED","md5":null}}`,
+	Iface:     "enp2s0f1",
+	HTTPHost:  "api.icndb.com",
+}
+
 var entries = []types.Entry{
 	types.Entry{
 		SrcIP:     "10.0.0.10",
@@ -127,6 +135,31 @@ func TestJSONParseEVEempty(t *testing.T) {
 	}
 }
 
+func TestJSONParseEVEwithnull(t *testing.T) {
+	f, err := os.Open("testdata/jsonparse_eve_nulls.json")
+	if err != nil {
+		t.Fatalf(err.Error())
+	}
+	scanner := bufio.NewScanner(f)
+	i := 0
+	var entry types.Entry
+	for scanner.Scan() {
+		json := scanner.Bytes()
+		e, err := ParseJSON(json)
+		if err != nil {
+			t.Fatalf(err.Error())
+		}
+		entry = e
+		i++
+	}
+	if i != 1 {
+		t.Fatalf("should parse only one entry, got %d", i)
+	}
+	if !reflect.DeepEqual(nullEntry, entry) {
+		t.Fatalf("entry %d parsed from JSON does not match expected value", i)
+	}
+}
+
 func TestGetSensorID(t *testing.T) {
 	sid, err := GetSensorID()
 	if err != nil {